2024-11-14 14:00:29,244 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-14 14:00:29,263 main DEBUG Took 0.017155 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 14:00:29,264 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 14:00:29,264 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 14:00:29,265 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 14:00:29,266 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:00:29,273 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 14:00:29,285 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:00:29,286 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:00:29,287 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:00:29,287 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:00:29,288 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:00:29,288 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:00:29,289 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:00:29,289 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:00:29,289 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:00:29,290 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:00:29,290 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:00:29,291 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:00:29,291 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:00:29,291 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:00:29,292 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:00:29,292 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:00:29,292 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:00:29,293 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:00:29,293 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:00:29,293 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:00:29,294 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:00:29,294 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:00:29,294 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:00:29,295 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:00:29,295 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:00:29,295 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 14:00:29,297 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:00:29,298 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 14:00:29,300 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 14:00:29,300 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 14:00:29,301 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 14:00:29,302 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 14:00:29,310 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 14:00:29,313 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 14:00:29,315 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 14:00:29,315 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 14:00:29,315 main DEBUG createAppenders(={Console}) 2024-11-14 14:00:29,316 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-11-14 14:00:29,316 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-14 14:00:29,317 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-11-14 14:00:29,317 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 14:00:29,318 main DEBUG OutputStream closed 2024-11-14 14:00:29,318 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 14:00:29,318 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 14:00:29,319 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-11-14 14:00:29,395 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 14:00:29,397 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 14:00:29,398 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 14:00:29,399 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 14:00:29,400 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 14:00:29,400 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 14:00:29,401 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 14:00:29,401 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 14:00:29,402 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 14:00:29,402 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 14:00:29,403 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 14:00:29,403 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 14:00:29,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 14:00:29,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 14:00:29,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 14:00:29,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 14:00:29,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 14:00:29,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 14:00:29,408 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 14:00:29,409 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4331d187) with optional ClassLoader: null 2024-11-14 14:00:29,409 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 14:00:29,410 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4331d187] started OK. 2024-11-14T14:00:29,429 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync timeout: 13 mins 2024-11-14 14:00:29,433 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 14:00:29,434 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T14:00:29,487 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=11, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=323, ProcessCount=11, AvailableMemoryMB=7693 2024-11-14T14:00:29,725 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c 2024-11-14T14:00:30,304 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T14:00:30,419 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T14:00:30,487 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=wal, suffix=, logDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync, archiveDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/oldWALs, maxLogs=55 2024-11-14T14:00:30,545 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592830533 2024-11-14T14:00:30,619 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592830533 2024-11-14T14:00:30,696 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:30,702 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-11-14T14:00:30,702 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-11-14T14:00:30,742 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => fb54b5db78ce062cb48717f016e99f5f, NAME => 'testLockupAroundBadAssignSync,,1731592830716.fb54b5db78ce062cb48717f016e99f5f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testLockupAroundBadAssignSync', {TABLE_ATTRIBUTES => {DURABILITY => 'SYNC_WAL', READONLY => 'false'}}, {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c 2024-11-14T14:00:30,802 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T14:00:30,806 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testLockupAroundBadAssignSync,,1731592830716.fb54b5db78ce062cb48717f016e99f5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:00:30,884 INFO [StoreOpener-fb54b5db78ce062cb48717f016e99f5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family MyCF of region fb54b5db78ce062cb48717f016e99f5f 2024-11-14T14:00:30,914 INFO [StoreOpener-fb54b5db78ce062cb48717f016e99f5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb54b5db78ce062cb48717f016e99f5f columnFamilyName MyCF 2024-11-14T14:00:30,920 DEBUG [StoreOpener-fb54b5db78ce062cb48717f016e99f5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:00:30,926 INFO [StoreOpener-fb54b5db78ce062cb48717f016e99f5f-1 {}] regionserver.HStore(327): Store=fb54b5db78ce062cb48717f016e99f5f/MyCF, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:00:30,930 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fb54b5db78ce062cb48717f016e99f5f 2024-11-14T14:00:30,936 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/data/default/testLockupAroundBadAssignSync/fb54b5db78ce062cb48717f016e99f5f 2024-11-14T14:00:30,937 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/data/default/testLockupAroundBadAssignSync/fb54b5db78ce062cb48717f016e99f5f 2024-11-14T14:00:30,939 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fb54b5db78ce062cb48717f016e99f5f 2024-11-14T14:00:30,940 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fb54b5db78ce062cb48717f016e99f5f 2024-11-14T14:00:30,955 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fb54b5db78ce062cb48717f016e99f5f 2024-11-14T14:00:30,980 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/data/default/testLockupAroundBadAssignSync/fb54b5db78ce062cb48717f016e99f5f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:00:30,981 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fb54b5db78ce062cb48717f016e99f5f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59524103, jitterRate=-0.11302174627780914}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:00:30,990 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T14:00:31,005 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fb54b5db78ce062cb48717f016e99f5f: Writing region info on filesystem at 1731592830853Initializing all the Stores at 1731592830855 (+2 ms)Instantiating store for column family {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}} at 1731592830855Cleaning up temporary data from old regions at 1731592830940 (+85 ms)Region opened successfully at 1731592831002 (+62 ms) 2024-11-14T14:00:31,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592830533) roll requested 2024-11-14T14:00:31,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831061 2024-11-14T14:00:31,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592830533 with entries=1, filesize=249 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831061 2024-11-14T14:00:31,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831061) roll requested 2024-11-14T14:00:31,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831086 2024-11-14T14:00:31,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831086 2024-11-14T14:00:31,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831086) roll requested 2024-11-14T14:00:31,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831102 2024-11-14T14:00:31,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831102 2024-11-14T14:00:31,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831102) roll requested 2024-11-14T14:00:31,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831122 2024-11-14T14:00:31,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831122 2024-11-14T14:00:31,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831122) roll requested 2024-11-14T14:00:31,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831142 2024-11-14T14:00:31,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831142 2024-11-14T14:00:31,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831142) roll requested 2024-11-14T14:00:31,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831155 2024-11-14T14:00:31,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831155 2024-11-14T14:00:31,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831155) roll requested 2024-11-14T14:00:31,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831166 2024-11-14T14:00:31,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831166 2024-11-14T14:00:31,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831166) roll requested 2024-11-14T14:00:31,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831177 2024-11-14T14:00:31,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831177 2024-11-14T14:00:31,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831177) roll requested 2024-11-14T14:00:31,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831188 2024-11-14T14:00:31,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831188 2024-11-14T14:00:31,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831188) roll requested 2024-11-14T14:00:31,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831198 2024-11-14T14:00:31,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831198 2024-11-14T14:00:31,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831198) roll requested 2024-11-14T14:00:31,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831209 2024-11-14T14:00:31,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831209 2024-11-14T14:00:31,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831209) roll requested 2024-11-14T14:00:31,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831220 2024-11-14T14:00:31,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831220 2024-11-14T14:00:31,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831220) roll requested 2024-11-14T14:00:31,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831230 2024-11-14T14:00:31,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831230 2024-11-14T14:00:31,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831230) roll requested 2024-11-14T14:00:31,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831240 2024-11-14T14:00:31,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831240 2024-11-14T14:00:31,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831240) roll requested 2024-11-14T14:00:31,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831250 2024-11-14T14:00:31,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831250 2024-11-14T14:00:31,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831250) roll requested 2024-11-14T14:00:31,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831260 2024-11-14T14:00:31,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831260 2024-11-14T14:00:31,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831260) roll requested 2024-11-14T14:00:31,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831270 2024-11-14T14:00:31,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831270 2024-11-14T14:00:31,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831270) roll requested 2024-11-14T14:00:31,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831285 2024-11-14T14:00:31,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831285 2024-11-14T14:00:31,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831285) roll requested 2024-11-14T14:00:31,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831295 2024-11-14T14:00:31,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831295 2024-11-14T14:00:31,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831295) roll requested 2024-11-14T14:00:31,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831305 2024-11-14T14:00:31,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831305 2024-11-14T14:00:31,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831305) roll requested 2024-11-14T14:00:31,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831320 2024-11-14T14:00:31,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831320 2024-11-14T14:00:31,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831320) roll requested 2024-11-14T14:00:31,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831330 2024-11-14T14:00:31,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831330 2024-11-14T14:00:31,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831330) roll requested 2024-11-14T14:00:31,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831342 2024-11-14T14:00:31,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831342 2024-11-14T14:00:31,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831342) roll requested 2024-11-14T14:00:31,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831353 2024-11-14T14:00:31,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831353 2024-11-14T14:00:31,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831353) roll requested 2024-11-14T14:00:31,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831362 2024-11-14T14:00:31,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831362 2024-11-14T14:00:31,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831362) roll requested 2024-11-14T14:00:31,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831371 2024-11-14T14:00:31,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831371 2024-11-14T14:00:31,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831371) roll requested 2024-11-14T14:00:31,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831380 2024-11-14T14:00:31,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831380 2024-11-14T14:00:31,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831380) roll requested 2024-11-14T14:00:31,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831390 2024-11-14T14:00:31,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831390 2024-11-14T14:00:31,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831390) roll requested 2024-11-14T14:00:31,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831398 2024-11-14T14:00:31,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831398 2024-11-14T14:00:31,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831398) roll requested 2024-11-14T14:00:31,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831409 2024-11-14T14:00:31,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831409 2024-11-14T14:00:31,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831409) roll requested 2024-11-14T14:00:31,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831418 2024-11-14T14:00:31,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831418 2024-11-14T14:00:31,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831418) roll requested 2024-11-14T14:00:31,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831430 2024-11-14T14:00:31,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831430 2024-11-14T14:00:31,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831430) roll requested 2024-11-14T14:00:31,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831440 2024-11-14T14:00:31,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831440 2024-11-14T14:00:31,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831440) roll requested 2024-11-14T14:00:31,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831449 2024-11-14T14:00:31,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831449 2024-11-14T14:00:31,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831449) roll requested 2024-11-14T14:00:31,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831457 2024-11-14T14:00:31,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831457 2024-11-14T14:00:31,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831457) roll requested 2024-11-14T14:00:31,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831465 2024-11-14T14:00:31,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831465 2024-11-14T14:00:31,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831465) roll requested 2024-11-14T14:00:31,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831474 2024-11-14T14:00:31,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831474 2024-11-14T14:00:31,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831474) roll requested 2024-11-14T14:00:31,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831483 2024-11-14T14:00:31,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831483 2024-11-14T14:00:31,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831483) roll requested 2024-11-14T14:00:31,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831492 2024-11-14T14:00:31,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831492 2024-11-14T14:00:31,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831492) roll requested 2024-11-14T14:00:31,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831503 2024-11-14T14:00:31,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831503 2024-11-14T14:00:31,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831503) roll requested 2024-11-14T14:00:31,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831511 2024-11-14T14:00:31,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831511 2024-11-14T14:00:31,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831511) roll requested 2024-11-14T14:00:31,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831520 2024-11-14T14:00:31,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831520 2024-11-14T14:00:31,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831520) roll requested 2024-11-14T14:00:31,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831530 2024-11-14T14:00:31,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831530 2024-11-14T14:00:31,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831530) roll requested 2024-11-14T14:00:31,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831539 2024-11-14T14:00:31,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831539 2024-11-14T14:00:31,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831539) roll requested 2024-11-14T14:00:31,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831547 2024-11-14T14:00:31,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831547 2024-11-14T14:00:31,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831547) roll requested 2024-11-14T14:00:31,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831555 2024-11-14T14:00:31,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831555 2024-11-14T14:00:31,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831555) roll requested 2024-11-14T14:00:31,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831564 2024-11-14T14:00:31,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831564 2024-11-14T14:00:31,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831564) roll requested 2024-11-14T14:00:31,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831572 2024-11-14T14:00:31,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831572 2024-11-14T14:00:31,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831572) roll requested 2024-11-14T14:00:31,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831580 2024-11-14T14:00:31,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831580 2024-11-14T14:00:31,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831580) roll requested 2024-11-14T14:00:31,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831589 2024-11-14T14:00:31,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831589 2024-11-14T14:00:31,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831589) roll requested 2024-11-14T14:00:31,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831597 2024-11-14T14:00:31,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831597 2024-11-14T14:00:31,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831597) roll requested 2024-11-14T14:00:31,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831607 2024-11-14T14:00:31,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831607 2024-11-14T14:00:31,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831607) roll requested 2024-11-14T14:00:31,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831617 2024-11-14T14:00:31,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831617 2024-11-14T14:00:31,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831617) roll requested 2024-11-14T14:00:31,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831627 2024-11-14T14:00:31,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831627 2024-11-14T14:00:31,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831627) roll requested 2024-11-14T14:00:31,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831637 2024-11-14T14:00:31,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831637 2024-11-14T14:00:31,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831637) roll requested 2024-11-14T14:00:31,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831645 2024-11-14T14:00:31,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831645 2024-11-14T14:00:31,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831645) roll requested 2024-11-14T14:00:31,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831656 2024-11-14T14:00:31,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831656 2024-11-14T14:00:31,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831656) roll requested 2024-11-14T14:00:31,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831664 2024-11-14T14:00:31,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831664 2024-11-14T14:00:31,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831664) roll requested 2024-11-14T14:00:31,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831672 2024-11-14T14:00:31,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831672 2024-11-14T14:00:31,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831672) roll requested 2024-11-14T14:00:31,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831680 2024-11-14T14:00:31,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831680 2024-11-14T14:00:31,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831680) roll requested 2024-11-14T14:00:31,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831690 2024-11-14T14:00:31,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831690 2024-11-14T14:00:31,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831690) roll requested 2024-11-14T14:00:31,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831698 2024-11-14T14:00:31,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831698 2024-11-14T14:00:31,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831698) roll requested 2024-11-14T14:00:31,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831706 2024-11-14T14:00:31,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831706 2024-11-14T14:00:31,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831706) roll requested 2024-11-14T14:00:31,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831720 2024-11-14T14:00:31,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831720 2024-11-14T14:00:31,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831720) roll requested 2024-11-14T14:00:31,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831731 2024-11-14T14:00:31,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831731 2024-11-14T14:00:31,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831731) roll requested 2024-11-14T14:00:31,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831739 2024-11-14T14:00:31,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831739 2024-11-14T14:00:31,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831739) roll requested 2024-11-14T14:00:31,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831746 2024-11-14T14:00:31,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831746 2024-11-14T14:00:31,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831746) roll requested 2024-11-14T14:00:31,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831754 2024-11-14T14:00:31,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831754 2024-11-14T14:00:31,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831754) roll requested 2024-11-14T14:00:31,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831764 2024-11-14T14:00:31,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831764 2024-11-14T14:00:31,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831764) roll requested 2024-11-14T14:00:31,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831772 2024-11-14T14:00:31,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831772 2024-11-14T14:00:31,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831772) roll requested 2024-11-14T14:00:31,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831780 2024-11-14T14:00:31,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831780 2024-11-14T14:00:31,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831780) roll requested 2024-11-14T14:00:31,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831787 2024-11-14T14:00:31,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831787 2024-11-14T14:00:31,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831787) roll requested 2024-11-14T14:00:31,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831795 2024-11-14T14:00:31,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831795 2024-11-14T14:00:31,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831795) roll requested 2024-11-14T14:00:31,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831803 2024-11-14T14:00:31,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831803 2024-11-14T14:00:31,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831803) roll requested 2024-11-14T14:00:31,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831810 2024-11-14T14:00:31,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831810 2024-11-14T14:00:31,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831810) roll requested 2024-11-14T14:00:31,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831818 2024-11-14T14:00:31,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831818 2024-11-14T14:00:31,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831818) roll requested 2024-11-14T14:00:31,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831826 2024-11-14T14:00:31,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831826 2024-11-14T14:00:31,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831826) roll requested 2024-11-14T14:00:31,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831834 2024-11-14T14:00:31,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831834 2024-11-14T14:00:31,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831834) roll requested 2024-11-14T14:00:31,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831841 2024-11-14T14:00:31,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831841 2024-11-14T14:00:31,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831841) roll requested 2024-11-14T14:00:31,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831849 2024-11-14T14:00:31,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831849 2024-11-14T14:00:31,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831849) roll requested 2024-11-14T14:00:31,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831857 2024-11-14T14:00:31,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831857 2024-11-14T14:00:31,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831857) roll requested 2024-11-14T14:00:31,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831865 2024-11-14T14:00:31,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831865 2024-11-14T14:00:31,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831865) roll requested 2024-11-14T14:00:31,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831872 2024-11-14T14:00:31,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831872 2024-11-14T14:00:31,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831872) roll requested 2024-11-14T14:00:31,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831879 2024-11-14T14:00:31,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831879 2024-11-14T14:00:31,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831879) roll requested 2024-11-14T14:00:31,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831887 2024-11-14T14:00:31,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831887 2024-11-14T14:00:31,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831887) roll requested 2024-11-14T14:00:31,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831895 2024-11-14T14:00:31,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831895 2024-11-14T14:00:31,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831895) roll requested 2024-11-14T14:00:31,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831903 2024-11-14T14:00:31,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831903 2024-11-14T14:00:31,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831903) roll requested 2024-11-14T14:00:31,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831911 2024-11-14T14:00:31,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831911 2024-11-14T14:00:31,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831911) roll requested 2024-11-14T14:00:31,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831919 2024-11-14T14:00:31,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831919 2024-11-14T14:00:31,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831919) roll requested 2024-11-14T14:00:31,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831926 2024-11-14T14:00:31,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831926 2024-11-14T14:00:31,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831926) roll requested 2024-11-14T14:00:31,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831933 2024-11-14T14:00:31,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831933 2024-11-14T14:00:31,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831933) roll requested 2024-11-14T14:00:31,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831941 2024-11-14T14:00:31,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831941 2024-11-14T14:00:31,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831941) roll requested 2024-11-14T14:00:31,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831949 2024-11-14T14:00:31,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831949 2024-11-14T14:00:31,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831949) roll requested 2024-11-14T14:00:31,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831956 2024-11-14T14:00:31,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831956 2024-11-14T14:00:31,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831956) roll requested 2024-11-14T14:00:31,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831963 2024-11-14T14:00:31,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831963 2024-11-14T14:00:31,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831963) roll requested 2024-11-14T14:00:31,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831970 2024-11-14T14:00:31,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831970 2024-11-14T14:00:31,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831970) roll requested 2024-11-14T14:00:31,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831978 2024-11-14T14:00:31,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831978 2024-11-14T14:00:31,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831978) roll requested 2024-11-14T14:00:31,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831988 2024-11-14T14:00:31,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:31,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831988 2024-11-14T14:00:31,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:31,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:31,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831988) roll requested 2024-11-14T14:00:31,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592831998 2024-11-14T14:00:32,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831998 2024-11-14T14:00:32,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592831998) roll requested 2024-11-14T14:00:32,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832010 2024-11-14T14:00:32,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592831998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832010 2024-11-14T14:00:32,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832010) roll requested 2024-11-14T14:00:32,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832022 2024-11-14T14:00:32,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832022 2024-11-14T14:00:32,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832022) roll requested 2024-11-14T14:00:32,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832041 2024-11-14T14:00:32,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832041 2024-11-14T14:00:32,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832041) roll requested 2024-11-14T14:00:32,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832053 2024-11-14T14:00:32,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832053 2024-11-14T14:00:32,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832053) roll requested 2024-11-14T14:00:32,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832062 2024-11-14T14:00:32,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832062 2024-11-14T14:00:32,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832062) roll requested 2024-11-14T14:00:32,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832070 2024-11-14T14:00:32,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832070 2024-11-14T14:00:32,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832070) roll requested 2024-11-14T14:00:32,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832082 2024-11-14T14:00:32,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832082 2024-11-14T14:00:32,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832082) roll requested 2024-11-14T14:00:32,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832095 2024-11-14T14:00:32,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832095 2024-11-14T14:00:32,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832095) roll requested 2024-11-14T14:00:32,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832105 2024-11-14T14:00:32,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832105 2024-11-14T14:00:32,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832105) roll requested 2024-11-14T14:00:32,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832113 2024-11-14T14:00:32,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832113 2024-11-14T14:00:32,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832113) roll requested 2024-11-14T14:00:32,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832121 2024-11-14T14:00:32,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832121 2024-11-14T14:00:32,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832121) roll requested 2024-11-14T14:00:32,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832129 2024-11-14T14:00:32,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832129 2024-11-14T14:00:32,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832129) roll requested 2024-11-14T14:00:32,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832141 2024-11-14T14:00:32,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832141 2024-11-14T14:00:32,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832141) roll requested 2024-11-14T14:00:32,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832149 2024-11-14T14:00:32,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832149 2024-11-14T14:00:32,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832149) roll requested 2024-11-14T14:00:32,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832156 2024-11-14T14:00:32,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832156 2024-11-14T14:00:32,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832156) roll requested 2024-11-14T14:00:32,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832163 2024-11-14T14:00:32,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832163 2024-11-14T14:00:32,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832163) roll requested 2024-11-14T14:00:32,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832171 2024-11-14T14:00:32,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832171 2024-11-14T14:00:32,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832171) roll requested 2024-11-14T14:00:32,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832185 2024-11-14T14:00:32,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832185 2024-11-14T14:00:32,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832185) roll requested 2024-11-14T14:00:32,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832193 2024-11-14T14:00:32,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832193 2024-11-14T14:00:32,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832193) roll requested 2024-11-14T14:00:32,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832200 2024-11-14T14:00:32,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832200 2024-11-14T14:00:32,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832200) roll requested 2024-11-14T14:00:32,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832208 2024-11-14T14:00:32,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832208 2024-11-14T14:00:32,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832208) roll requested 2024-11-14T14:00:32,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832217 2024-11-14T14:00:32,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832217 2024-11-14T14:00:32,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832217) roll requested 2024-11-14T14:00:32,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832225 2024-11-14T14:00:32,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832225 2024-11-14T14:00:32,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832225) roll requested 2024-11-14T14:00:32,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832232 2024-11-14T14:00:32,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832232 2024-11-14T14:00:32,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832232) roll requested 2024-11-14T14:00:32,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832240 2024-11-14T14:00:32,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832240 2024-11-14T14:00:32,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832240) roll requested 2024-11-14T14:00:32,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832249 2024-11-14T14:00:32,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832249 2024-11-14T14:00:32,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832249) roll requested 2024-11-14T14:00:32,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832261 2024-11-14T14:00:32,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832261 2024-11-14T14:00:32,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832261) roll requested 2024-11-14T14:00:32,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832268 2024-11-14T14:00:32,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832268 2024-11-14T14:00:32,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832268) roll requested 2024-11-14T14:00:32,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832280 2024-11-14T14:00:32,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832280 2024-11-14T14:00:32,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832280) roll requested 2024-11-14T14:00:32,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832289 2024-11-14T14:00:32,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832289 2024-11-14T14:00:32,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832289) roll requested 2024-11-14T14:00:32,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832297 2024-11-14T14:00:32,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832297 2024-11-14T14:00:32,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832297) roll requested 2024-11-14T14:00:32,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832305 2024-11-14T14:00:32,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832305 2024-11-14T14:00:32,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832305) roll requested 2024-11-14T14:00:32,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832312 2024-11-14T14:00:32,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832312 2024-11-14T14:00:32,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832312) roll requested 2024-11-14T14:00:32,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832319 2024-11-14T14:00:32,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832319 2024-11-14T14:00:32,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832319) roll requested 2024-11-14T14:00:32,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832327 2024-11-14T14:00:32,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832327 2024-11-14T14:00:32,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832327) roll requested 2024-11-14T14:00:32,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832334 2024-11-14T14:00:32,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832334 2024-11-14T14:00:32,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832334) roll requested 2024-11-14T14:00:32,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832344 2024-11-14T14:00:32,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832344 2024-11-14T14:00:32,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832344) roll requested 2024-11-14T14:00:32,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832352 2024-11-14T14:00:32,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832352 2024-11-14T14:00:32,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832352) roll requested 2024-11-14T14:00:32,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832359 2024-11-14T14:00:32,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832359 2024-11-14T14:00:32,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832359) roll requested 2024-11-14T14:00:32,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832367 2024-11-14T14:00:32,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832367 2024-11-14T14:00:32,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832367) roll requested 2024-11-14T14:00:32,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832374 2024-11-14T14:00:32,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832374 2024-11-14T14:00:32,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832374) roll requested 2024-11-14T14:00:32,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832382 2024-11-14T14:00:32,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832382 2024-11-14T14:00:32,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832382) roll requested 2024-11-14T14:00:32,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832389 2024-11-14T14:00:32,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832389 2024-11-14T14:00:32,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832389) roll requested 2024-11-14T14:00:32,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832397 2024-11-14T14:00:32,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832397 2024-11-14T14:00:32,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832397) roll requested 2024-11-14T14:00:32,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832405 2024-11-14T14:00:32,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832405 2024-11-14T14:00:32,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832405) roll requested 2024-11-14T14:00:32,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832413 2024-11-14T14:00:32,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832413 2024-11-14T14:00:32,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832413) roll requested 2024-11-14T14:00:32,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832424 2024-11-14T14:00:32,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832424 2024-11-14T14:00:32,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832424) roll requested 2024-11-14T14:00:32,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832433 2024-11-14T14:00:32,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832433 2024-11-14T14:00:32,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832433) roll requested 2024-11-14T14:00:32,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832442 2024-11-14T14:00:32,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832442 2024-11-14T14:00:32,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832442) roll requested 2024-11-14T14:00:32,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832449 2024-11-14T14:00:32,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832449 2024-11-14T14:00:32,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832449) roll requested 2024-11-14T14:00:32,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832457 2024-11-14T14:00:32,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832457 2024-11-14T14:00:32,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832457) roll requested 2024-11-14T14:00:32,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832466 2024-11-14T14:00:32,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832466 2024-11-14T14:00:32,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832466) roll requested 2024-11-14T14:00:32,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832478 2024-11-14T14:00:32,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832478 2024-11-14T14:00:32,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832478) roll requested 2024-11-14T14:00:32,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832487 2024-11-14T14:00:32,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832487 2024-11-14T14:00:32,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832487) roll requested 2024-11-14T14:00:32,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832496 2024-11-14T14:00:32,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832496 2024-11-14T14:00:32,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832496) roll requested 2024-11-14T14:00:32,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832505 2024-11-14T14:00:32,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832505 2024-11-14T14:00:32,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832505) roll requested 2024-11-14T14:00:32,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832514 2024-11-14T14:00:32,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832514 2024-11-14T14:00:32,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832514) roll requested 2024-11-14T14:00:32,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832522 2024-11-14T14:00:32,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832522 2024-11-14T14:00:32,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832522) roll requested 2024-11-14T14:00:32,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832530 2024-11-14T14:00:32,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832530 2024-11-14T14:00:32,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832530) roll requested 2024-11-14T14:00:32,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832541 2024-11-14T14:00:32,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832541 2024-11-14T14:00:32,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832541) roll requested 2024-11-14T14:00:32,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832549 2024-11-14T14:00:32,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832549 2024-11-14T14:00:32,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832549) roll requested 2024-11-14T14:00:32,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832559 2024-11-14T14:00:32,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832559 2024-11-14T14:00:32,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832559) roll requested 2024-11-14T14:00:32,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832567 2024-11-14T14:00:32,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832567 2024-11-14T14:00:32,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832567) roll requested 2024-11-14T14:00:32,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832576 2024-11-14T14:00:32,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832576 2024-11-14T14:00:32,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832576) roll requested 2024-11-14T14:00:32,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832584 2024-11-14T14:00:32,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832584 2024-11-14T14:00:32,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832584) roll requested 2024-11-14T14:00:32,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832591 2024-11-14T14:00:32,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832591 2024-11-14T14:00:32,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832591) roll requested 2024-11-14T14:00:32,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832599 2024-11-14T14:00:32,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832599 2024-11-14T14:00:32,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832599) roll requested 2024-11-14T14:00:32,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832609 2024-11-14T14:00:32,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832609 2024-11-14T14:00:32,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832609) roll requested 2024-11-14T14:00:32,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832617 2024-11-14T14:00:32,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832617 2024-11-14T14:00:32,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832617) roll requested 2024-11-14T14:00:32,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832624 2024-11-14T14:00:32,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832624 2024-11-14T14:00:32,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832624) roll requested 2024-11-14T14:00:32,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832635 2024-11-14T14:00:32,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832635 2024-11-14T14:00:32,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832635) roll requested 2024-11-14T14:00:32,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832645 2024-11-14T14:00:32,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832645 2024-11-14T14:00:32,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832645) roll requested 2024-11-14T14:00:32,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832652 2024-11-14T14:00:32,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832652 2024-11-14T14:00:32,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832652) roll requested 2024-11-14T14:00:32,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832660 2024-11-14T14:00:32,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832660 2024-11-14T14:00:32,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832660) roll requested 2024-11-14T14:00:32,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832667 2024-11-14T14:00:32,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832667 2024-11-14T14:00:32,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832667) roll requested 2024-11-14T14:00:32,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832675 2024-11-14T14:00:32,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832675 2024-11-14T14:00:32,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832675) roll requested 2024-11-14T14:00:32,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832682 2024-11-14T14:00:32,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832682 2024-11-14T14:00:32,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832682) roll requested 2024-11-14T14:00:32,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832691 2024-11-14T14:00:32,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832691 2024-11-14T14:00:32,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832691) roll requested 2024-11-14T14:00:32,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832700 2024-11-14T14:00:32,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832700 2024-11-14T14:00:32,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832700) roll requested 2024-11-14T14:00:32,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832709 2024-11-14T14:00:32,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832709 2024-11-14T14:00:32,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832709) roll requested 2024-11-14T14:00:32,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832716 2024-11-14T14:00:32,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832716 2024-11-14T14:00:32,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832716) roll requested 2024-11-14T14:00:32,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832725 2024-11-14T14:00:32,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832725 2024-11-14T14:00:32,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832725) roll requested 2024-11-14T14:00:32,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832732 2024-11-14T14:00:32,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832732 2024-11-14T14:00:32,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832732) roll requested 2024-11-14T14:00:32,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832741 2024-11-14T14:00:32,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832741 2024-11-14T14:00:32,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832741) roll requested 2024-11-14T14:00:32,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832752 2024-11-14T14:00:32,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832752 2024-11-14T14:00:32,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832752) roll requested 2024-11-14T14:00:32,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832761 2024-11-14T14:00:32,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832761 2024-11-14T14:00:32,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832761) roll requested 2024-11-14T14:00:32,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832768 2024-11-14T14:00:32,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832768 2024-11-14T14:00:32,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832768) roll requested 2024-11-14T14:00:32,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832775 2024-11-14T14:00:32,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832775 2024-11-14T14:00:32,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832775) roll requested 2024-11-14T14:00:32,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832783 2024-11-14T14:00:32,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832783 2024-11-14T14:00:32,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832783) roll requested 2024-11-14T14:00:32,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832790 2024-11-14T14:00:32,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832790 2024-11-14T14:00:32,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832790) roll requested 2024-11-14T14:00:32,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832797 2024-11-14T14:00:32,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832797 2024-11-14T14:00:32,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832797) roll requested 2024-11-14T14:00:32,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832811 2024-11-14T14:00:32,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832811 2024-11-14T14:00:32,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832811) roll requested 2024-11-14T14:00:32,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832819 2024-11-14T14:00:32,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832819 2024-11-14T14:00:32,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832819) roll requested 2024-11-14T14:00:32,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832827 2024-11-14T14:00:32,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832827 2024-11-14T14:00:32,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832827) roll requested 2024-11-14T14:00:32,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832834 2024-11-14T14:00:32,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832834 2024-11-14T14:00:32,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832834) roll requested 2024-11-14T14:00:32,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832842 2024-11-14T14:00:32,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832842 2024-11-14T14:00:32,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832842) roll requested 2024-11-14T14:00:32,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832852 2024-11-14T14:00:32,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832852 2024-11-14T14:00:32,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832852) roll requested 2024-11-14T14:00:32,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832861 2024-11-14T14:00:32,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832861 2024-11-14T14:00:32,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832861) roll requested 2024-11-14T14:00:32,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832868 2024-11-14T14:00:32,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832868 2024-11-14T14:00:32,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832868) roll requested 2024-11-14T14:00:32,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832876 2024-11-14T14:00:32,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832876 2024-11-14T14:00:32,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832876) roll requested 2024-11-14T14:00:32,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832883 2024-11-14T14:00:32,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832883 2024-11-14T14:00:32,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832883) roll requested 2024-11-14T14:00:32,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832893 2024-11-14T14:00:32,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832893 2024-11-14T14:00:32,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832893) roll requested 2024-11-14T14:00:32,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832900 2024-11-14T14:00:32,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832900 2024-11-14T14:00:32,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832900) roll requested 2024-11-14T14:00:32,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832908 2024-11-14T14:00:32,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832908 2024-11-14T14:00:32,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832908) roll requested 2024-11-14T14:00:32,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832917 2024-11-14T14:00:32,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832917 2024-11-14T14:00:32,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832917) roll requested 2024-11-14T14:00:32,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832925 2024-11-14T14:00:32,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832925 2024-11-14T14:00:32,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832925) roll requested 2024-11-14T14:00:32,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832936 2024-11-14T14:00:32,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832936 2024-11-14T14:00:32,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832936) roll requested 2024-11-14T14:00:32,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832944 2024-11-14T14:00:32,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832944 2024-11-14T14:00:32,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832944) roll requested 2024-11-14T14:00:32,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832962 2024-11-14T14:00:32,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832962 2024-11-14T14:00:32,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832962) roll requested 2024-11-14T14:00:32,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832975 2024-11-14T14:00:32,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832975 2024-11-14T14:00:32,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832975) roll requested 2024-11-14T14:00:32,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832985 2024-11-14T14:00:32,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:32,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832985 2024-11-14T14:00:32,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:32,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:32,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832985) roll requested 2024-11-14T14:00:32,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592832997 2024-11-14T14:00:33,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832997 2024-11-14T14:00:33,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592832997) roll requested 2024-11-14T14:00:33,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833006 2024-11-14T14:00:33,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592832997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833006 2024-11-14T14:00:33,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833006) roll requested 2024-11-14T14:00:33,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833025 2024-11-14T14:00:33,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833025 2024-11-14T14:00:33,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833025) roll requested 2024-11-14T14:00:33,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833037 2024-11-14T14:00:33,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833037 2024-11-14T14:00:33,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833037) roll requested 2024-11-14T14:00:33,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833046 2024-11-14T14:00:33,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833046 2024-11-14T14:00:33,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833046) roll requested 2024-11-14T14:00:33,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833055 2024-11-14T14:00:33,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833055 2024-11-14T14:00:33,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833055) roll requested 2024-11-14T14:00:33,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833066 2024-11-14T14:00:33,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833066 2024-11-14T14:00:33,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833066) roll requested 2024-11-14T14:00:33,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833075 2024-11-14T14:00:33,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833075 2024-11-14T14:00:33,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833075) roll requested 2024-11-14T14:00:33,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833093 2024-11-14T14:00:33,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833093 2024-11-14T14:00:33,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833093) roll requested 2024-11-14T14:00:33,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833102 2024-11-14T14:00:33,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833102 2024-11-14T14:00:33,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833102) roll requested 2024-11-14T14:00:33,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833111 2024-11-14T14:00:33,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833111 2024-11-14T14:00:33,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833111) roll requested 2024-11-14T14:00:33,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833120 2024-11-14T14:00:33,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833120 2024-11-14T14:00:33,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833120) roll requested 2024-11-14T14:00:33,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833129 2024-11-14T14:00:33,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833129 2024-11-14T14:00:33,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833129) roll requested 2024-11-14T14:00:33,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833138 2024-11-14T14:00:33,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833138 2024-11-14T14:00:33,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833138) roll requested 2024-11-14T14:00:33,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833150 2024-11-14T14:00:33,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833150 2024-11-14T14:00:33,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833150) roll requested 2024-11-14T14:00:33,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833161 2024-11-14T14:00:33,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833161 2024-11-14T14:00:33,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833161) roll requested 2024-11-14T14:00:33,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833175 2024-11-14T14:00:33,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833175 2024-11-14T14:00:33,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833175) roll requested 2024-11-14T14:00:33,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833183 2024-11-14T14:00:33,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833183 2024-11-14T14:00:33,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833183) roll requested 2024-11-14T14:00:33,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833193 2024-11-14T14:00:33,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833193 2024-11-14T14:00:33,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833193) roll requested 2024-11-14T14:00:33,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833205 2024-11-14T14:00:33,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833205 2024-11-14T14:00:33,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833205) roll requested 2024-11-14T14:00:33,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833213 2024-11-14T14:00:33,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833213 2024-11-14T14:00:33,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833213) roll requested 2024-11-14T14:00:33,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833229 2024-11-14T14:00:33,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833229 2024-11-14T14:00:33,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833229) roll requested 2024-11-14T14:00:33,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833238 2024-11-14T14:00:33,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833238 2024-11-14T14:00:33,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833238) roll requested 2024-11-14T14:00:33,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833245 2024-11-14T14:00:33,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833245 2024-11-14T14:00:33,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833245) roll requested 2024-11-14T14:00:33,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833254 2024-11-14T14:00:33,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833254 2024-11-14T14:00:33,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833254) roll requested 2024-11-14T14:00:33,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833262 2024-11-14T14:00:33,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833262 2024-11-14T14:00:33,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833262) roll requested 2024-11-14T14:00:33,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833272 2024-11-14T14:00:33,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833272 2024-11-14T14:00:33,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833272) roll requested 2024-11-14T14:00:33,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833289 2024-11-14T14:00:33,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833289 2024-11-14T14:00:33,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833289) roll requested 2024-11-14T14:00:33,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833302 2024-11-14T14:00:33,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833302 2024-11-14T14:00:33,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833302) roll requested 2024-11-14T14:00:33,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833310 2024-11-14T14:00:33,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833310 2024-11-14T14:00:33,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833310) roll requested 2024-11-14T14:00:33,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833319 2024-11-14T14:00:33,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833319 2024-11-14T14:00:33,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833319) roll requested 2024-11-14T14:00:33,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833327 2024-11-14T14:00:33,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833327 2024-11-14T14:00:33,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833327) roll requested 2024-11-14T14:00:33,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833335 2024-11-14T14:00:33,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833335 2024-11-14T14:00:33,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833335) roll requested 2024-11-14T14:00:33,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833344 2024-11-14T14:00:33,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833344 2024-11-14T14:00:33,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833344) roll requested 2024-11-14T14:00:33,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833356 2024-11-14T14:00:33,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833356 2024-11-14T14:00:33,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833356) roll requested 2024-11-14T14:00:33,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833364 2024-11-14T14:00:33,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833364 2024-11-14T14:00:33,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833364) roll requested 2024-11-14T14:00:33,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833373 2024-11-14T14:00:33,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833373 2024-11-14T14:00:33,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833373) roll requested 2024-11-14T14:00:33,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833387 2024-11-14T14:00:33,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833387 2024-11-14T14:00:33,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833387) roll requested 2024-11-14T14:00:33,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833395 2024-11-14T14:00:33,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833395 2024-11-14T14:00:33,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833395) roll requested 2024-11-14T14:00:33,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833414 2024-11-14T14:00:33,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833414 2024-11-14T14:00:33,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833414) roll requested 2024-11-14T14:00:33,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833425 2024-11-14T14:00:33,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833425 2024-11-14T14:00:33,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833425) roll requested 2024-11-14T14:00:33,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833435 2024-11-14T14:00:33,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833435 2024-11-14T14:00:33,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833435) roll requested 2024-11-14T14:00:33,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833446 2024-11-14T14:00:33,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833446 2024-11-14T14:00:33,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833446) roll requested 2024-11-14T14:00:33,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833456 2024-11-14T14:00:33,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833456 2024-11-14T14:00:33,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833456) roll requested 2024-11-14T14:00:33,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833467 2024-11-14T14:00:33,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833467 2024-11-14T14:00:33,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833467) roll requested 2024-11-14T14:00:33,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833480 2024-11-14T14:00:33,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833480 2024-11-14T14:00:33,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833480) roll requested 2024-11-14T14:00:33,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833491 2024-11-14T14:00:33,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833491 2024-11-14T14:00:33,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833491) roll requested 2024-11-14T14:00:33,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833500 2024-11-14T14:00:33,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833500 2024-11-14T14:00:33,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833500) roll requested 2024-11-14T14:00:33,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833508 2024-11-14T14:00:33,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833508 2024-11-14T14:00:33,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833508) roll requested 2024-11-14T14:00:33,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833517 2024-11-14T14:00:33,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833517 2024-11-14T14:00:33,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833517) roll requested 2024-11-14T14:00:33,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833525 2024-11-14T14:00:33,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833525 2024-11-14T14:00:33,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833525) roll requested 2024-11-14T14:00:33,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833535 2024-11-14T14:00:33,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833535 2024-11-14T14:00:33,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833535) roll requested 2024-11-14T14:00:33,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833548 2024-11-14T14:00:33,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833548 2024-11-14T14:00:33,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833548) roll requested 2024-11-14T14:00:33,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833557 2024-11-14T14:00:33,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833557 2024-11-14T14:00:33,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833557) roll requested 2024-11-14T14:00:33,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833565 2024-11-14T14:00:33,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833565 2024-11-14T14:00:33,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833565) roll requested 2024-11-14T14:00:33,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833575 2024-11-14T14:00:33,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833575 2024-11-14T14:00:33,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833575) roll requested 2024-11-14T14:00:33,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833585 2024-11-14T14:00:33,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833585 2024-11-14T14:00:33,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833585) roll requested 2024-11-14T14:00:33,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833593 2024-11-14T14:00:33,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833593 2024-11-14T14:00:33,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833593) roll requested 2024-11-14T14:00:33,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833602 2024-11-14T14:00:33,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833602 2024-11-14T14:00:33,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833602) roll requested 2024-11-14T14:00:33,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833609 2024-11-14T14:00:33,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833609 2024-11-14T14:00:33,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833609) roll requested 2024-11-14T14:00:33,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833617 2024-11-14T14:00:33,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833617 2024-11-14T14:00:33,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833617) roll requested 2024-11-14T14:00:33,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833624 2024-11-14T14:00:33,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833624 2024-11-14T14:00:33,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833624) roll requested 2024-11-14T14:00:33,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833632 2024-11-14T14:00:33,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833632 2024-11-14T14:00:33,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833632) roll requested 2024-11-14T14:00:33,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833639 2024-11-14T14:00:33,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833639 2024-11-14T14:00:33,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833639) roll requested 2024-11-14T14:00:33,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833647 2024-11-14T14:00:33,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833647 2024-11-14T14:00:33,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833647) roll requested 2024-11-14T14:00:33,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833657 2024-11-14T14:00:33,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833657 2024-11-14T14:00:33,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833657) roll requested 2024-11-14T14:00:33,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833665 2024-11-14T14:00:33,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833665 2024-11-14T14:00:33,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833665) roll requested 2024-11-14T14:00:33,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833675 2024-11-14T14:00:33,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833675 2024-11-14T14:00:33,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833675) roll requested 2024-11-14T14:00:33,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833682 2024-11-14T14:00:33,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833682 2024-11-14T14:00:33,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833682) roll requested 2024-11-14T14:00:33,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833690 2024-11-14T14:00:33,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833690 2024-11-14T14:00:33,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833690) roll requested 2024-11-14T14:00:33,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833703 2024-11-14T14:00:33,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833703 2024-11-14T14:00:33,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833703) roll requested 2024-11-14T14:00:33,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833711 2024-11-14T14:00:33,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833711 2024-11-14T14:00:33,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833711) roll requested 2024-11-14T14:00:33,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833719 2024-11-14T14:00:33,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833719 2024-11-14T14:00:33,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833719) roll requested 2024-11-14T14:00:33,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833727 2024-11-14T14:00:33,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833727 2024-11-14T14:00:33,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833727) roll requested 2024-11-14T14:00:33,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833740 2024-11-14T14:00:33,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833740 2024-11-14T14:00:33,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833740) roll requested 2024-11-14T14:00:33,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833761 2024-11-14T14:00:33,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833761 2024-11-14T14:00:33,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833761) roll requested 2024-11-14T14:00:33,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833769 2024-11-14T14:00:33,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833769 2024-11-14T14:00:33,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833769) roll requested 2024-11-14T14:00:33,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833780 2024-11-14T14:00:33,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833780 2024-11-14T14:00:33,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833780) roll requested 2024-11-14T14:00:33,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833788 2024-11-14T14:00:33,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833788 2024-11-14T14:00:33,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833788) roll requested 2024-11-14T14:00:33,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833797 2024-11-14T14:00:33,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833797 2024-11-14T14:00:33,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833797) roll requested 2024-11-14T14:00:33,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833805 2024-11-14T14:00:33,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833805 2024-11-14T14:00:33,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833805) roll requested 2024-11-14T14:00:33,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833812 2024-11-14T14:00:33,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833812 2024-11-14T14:00:33,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833812) roll requested 2024-11-14T14:00:33,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833825 2024-11-14T14:00:33,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833825 2024-11-14T14:00:33,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833825) roll requested 2024-11-14T14:00:33,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833833 2024-11-14T14:00:33,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833833 2024-11-14T14:00:33,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833833) roll requested 2024-11-14T14:00:33,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833840 2024-11-14T14:00:33,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833840 2024-11-14T14:00:33,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833840) roll requested 2024-11-14T14:00:33,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833849 2024-11-14T14:00:33,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833849 2024-11-14T14:00:33,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833849) roll requested 2024-11-14T14:00:33,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833861 2024-11-14T14:00:33,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833861 2024-11-14T14:00:33,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833861) roll requested 2024-11-14T14:00:33,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833877 2024-11-14T14:00:33,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833877 2024-11-14T14:00:33,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833877) roll requested 2024-11-14T14:00:33,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833885 2024-11-14T14:00:33,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833885 2024-11-14T14:00:33,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833885) roll requested 2024-11-14T14:00:33,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833893 2024-11-14T14:00:33,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833893 2024-11-14T14:00:33,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833893) roll requested 2024-11-14T14:00:33,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833900 2024-11-14T14:00:33,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833900 2024-11-14T14:00:33,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833900) roll requested 2024-11-14T14:00:33,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833909 2024-11-14T14:00:33,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833909 2024-11-14T14:00:33,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833909) roll requested 2024-11-14T14:00:33,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833917 2024-11-14T14:00:33,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833917 2024-11-14T14:00:33,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833917) roll requested 2024-11-14T14:00:33,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833924 2024-11-14T14:00:33,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833924 2024-11-14T14:00:33,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833924) roll requested 2024-11-14T14:00:33,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833934 2024-11-14T14:00:33,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833934 2024-11-14T14:00:33,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833934) roll requested 2024-11-14T14:00:33,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833942 2024-11-14T14:00:33,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833942 2024-11-14T14:00:33,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833942) roll requested 2024-11-14T14:00:33,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833950 2024-11-14T14:00:33,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833950 2024-11-14T14:00:33,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833950) roll requested 2024-11-14T14:00:33,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833960 2024-11-14T14:00:33,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833960 2024-11-14T14:00:33,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833960) roll requested 2024-11-14T14:00:33,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833967 2024-11-14T14:00:33,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833967 2024-11-14T14:00:33,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833967) roll requested 2024-11-14T14:00:33,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833974 2024-11-14T14:00:33,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833974 2024-11-14T14:00:33,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833974) roll requested 2024-11-14T14:00:33,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833982 2024-11-14T14:00:33,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:33,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833982 2024-11-14T14:00:33,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:33,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:33,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833982) roll requested 2024-11-14T14:00:33,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592833990 2024-11-14T14:00:34,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833990 2024-11-14T14:00:34,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592833990) roll requested 2024-11-14T14:00:34,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834003 2024-11-14T14:00:34,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592833990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834003 2024-11-14T14:00:34,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834003) roll requested 2024-11-14T14:00:34,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834010 2024-11-14T14:00:34,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834010 2024-11-14T14:00:34,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834010) roll requested 2024-11-14T14:00:34,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834021 2024-11-14T14:00:34,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834021 2024-11-14T14:00:34,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834021) roll requested 2024-11-14T14:00:34,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834033 2024-11-14T14:00:34,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834033 2024-11-14T14:00:34,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834033) roll requested 2024-11-14T14:00:34,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834040 2024-11-14T14:00:34,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834040 2024-11-14T14:00:34,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834040) roll requested 2024-11-14T14:00:34,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834050 2024-11-14T14:00:34,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834050 2024-11-14T14:00:34,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834050) roll requested 2024-11-14T14:00:34,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834058 2024-11-14T14:00:34,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834058 2024-11-14T14:00:34,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834058) roll requested 2024-11-14T14:00:34,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834066 2024-11-14T14:00:34,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834066 2024-11-14T14:00:34,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834066) roll requested 2024-11-14T14:00:34,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834074 2024-11-14T14:00:34,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834074 2024-11-14T14:00:34,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834074) roll requested 2024-11-14T14:00:34,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834081 2024-11-14T14:00:34,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834081 2024-11-14T14:00:34,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834081) roll requested 2024-11-14T14:00:34,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834088 2024-11-14T14:00:34,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834088 2024-11-14T14:00:34,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834088) roll requested 2024-11-14T14:00:34,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834096 2024-11-14T14:00:34,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834096 2024-11-14T14:00:34,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834096) roll requested 2024-11-14T14:00:34,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834106 2024-11-14T14:00:34,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834106 2024-11-14T14:00:34,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834106) roll requested 2024-11-14T14:00:34,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834113 2024-11-14T14:00:34,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834113 2024-11-14T14:00:34,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834113) roll requested 2024-11-14T14:00:34,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834122 2024-11-14T14:00:34,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834122 2024-11-14T14:00:34,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834122) roll requested 2024-11-14T14:00:34,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834130 2024-11-14T14:00:34,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834130 2024-11-14T14:00:34,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834130) roll requested 2024-11-14T14:00:34,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834137 2024-11-14T14:00:34,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834137 2024-11-14T14:00:34,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834137) roll requested 2024-11-14T14:00:34,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834145 2024-11-14T14:00:34,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834145 2024-11-14T14:00:34,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834145) roll requested 2024-11-14T14:00:34,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834154 2024-11-14T14:00:34,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834154 2024-11-14T14:00:34,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834154) roll requested 2024-11-14T14:00:34,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834161 2024-11-14T14:00:34,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834161 2024-11-14T14:00:34,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834161) roll requested 2024-11-14T14:00:34,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834168 2024-11-14T14:00:34,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834168 2024-11-14T14:00:34,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834168) roll requested 2024-11-14T14:00:34,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834179 2024-11-14T14:00:34,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834179 2024-11-14T14:00:34,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834179) roll requested 2024-11-14T14:00:34,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834187 2024-11-14T14:00:34,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834187 2024-11-14T14:00:34,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834187) roll requested 2024-11-14T14:00:34,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834196 2024-11-14T14:00:34,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834196 2024-11-14T14:00:34,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834196) roll requested 2024-11-14T14:00:34,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834204 2024-11-14T14:00:34,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834204 2024-11-14T14:00:34,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834204) roll requested 2024-11-14T14:00:34,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834212 2024-11-14T14:00:34,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834212 2024-11-14T14:00:34,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834212) roll requested 2024-11-14T14:00:34,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834220 2024-11-14T14:00:34,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834220 2024-11-14T14:00:34,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834220) roll requested 2024-11-14T14:00:34,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834228 2024-11-14T14:00:34,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834228 2024-11-14T14:00:34,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834228) roll requested 2024-11-14T14:00:34,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834235 2024-11-14T14:00:34,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834235 2024-11-14T14:00:34,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834235) roll requested 2024-11-14T14:00:34,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834244 2024-11-14T14:00:34,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834244 2024-11-14T14:00:34,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834244) roll requested 2024-11-14T14:00:34,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834251 2024-11-14T14:00:34,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834251 2024-11-14T14:00:34,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834251) roll requested 2024-11-14T14:00:34,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834260 2024-11-14T14:00:34,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834260 2024-11-14T14:00:34,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834260) roll requested 2024-11-14T14:00:34,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834267 2024-11-14T14:00:34,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834267 2024-11-14T14:00:34,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834267) roll requested 2024-11-14T14:00:34,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834277 2024-11-14T14:00:34,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834277 2024-11-14T14:00:34,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834277) roll requested 2024-11-14T14:00:34,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834286 2024-11-14T14:00:34,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834286 2024-11-14T14:00:34,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834286) roll requested 2024-11-14T14:00:34,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834294 2024-11-14T14:00:34,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834294 2024-11-14T14:00:34,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834294) roll requested 2024-11-14T14:00:34,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834303 2024-11-14T14:00:34,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834303 2024-11-14T14:00:34,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834303) roll requested 2024-11-14T14:00:34,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834314 2024-11-14T14:00:34,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834314 2024-11-14T14:00:34,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834314) roll requested 2024-11-14T14:00:34,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834322 2024-11-14T14:00:34,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834322 2024-11-14T14:00:34,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834322) roll requested 2024-11-14T14:00:34,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834329 2024-11-14T14:00:34,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834329 2024-11-14T14:00:34,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834329) roll requested 2024-11-14T14:00:34,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834336 2024-11-14T14:00:34,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834336 2024-11-14T14:00:34,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834336) roll requested 2024-11-14T14:00:34,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834343 2024-11-14T14:00:34,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834343 2024-11-14T14:00:34,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834343) roll requested 2024-11-14T14:00:34,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834350 2024-11-14T14:00:34,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834350 2024-11-14T14:00:34,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834350) roll requested 2024-11-14T14:00:34,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834359 2024-11-14T14:00:34,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834359 2024-11-14T14:00:34,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834359) roll requested 2024-11-14T14:00:34,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834368 2024-11-14T14:00:34,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834368 2024-11-14T14:00:34,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834368) roll requested 2024-11-14T14:00:34,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834374 2024-11-14T14:00:34,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834374 2024-11-14T14:00:34,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834374) roll requested 2024-11-14T14:00:34,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834381 2024-11-14T14:00:34,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834381 2024-11-14T14:00:34,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834381) roll requested 2024-11-14T14:00:34,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834390 2024-11-14T14:00:34,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834390 2024-11-14T14:00:34,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834390) roll requested 2024-11-14T14:00:34,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834398 2024-11-14T14:00:34,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834398 2024-11-14T14:00:34,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834398) roll requested 2024-11-14T14:00:34,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834406 2024-11-14T14:00:34,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834406 2024-11-14T14:00:34,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834406) roll requested 2024-11-14T14:00:34,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834413 2024-11-14T14:00:34,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834413 2024-11-14T14:00:34,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834413) roll requested 2024-11-14T14:00:34,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834420 2024-11-14T14:00:34,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834420 2024-11-14T14:00:34,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834420) roll requested 2024-11-14T14:00:34,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834427 2024-11-14T14:00:34,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834427 2024-11-14T14:00:34,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834427) roll requested 2024-11-14T14:00:34,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834434 2024-11-14T14:00:34,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834434 2024-11-14T14:00:34,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834434) roll requested 2024-11-14T14:00:34,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834441 2024-11-14T14:00:34,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834441 2024-11-14T14:00:34,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834441) roll requested 2024-11-14T14:00:34,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834448 2024-11-14T14:00:34,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834448 2024-11-14T14:00:34,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834448) roll requested 2024-11-14T14:00:34,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834455 2024-11-14T14:00:34,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834455 2024-11-14T14:00:34,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834455) roll requested 2024-11-14T14:00:34,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834462 2024-11-14T14:00:34,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834462 2024-11-14T14:00:34,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834462) roll requested 2024-11-14T14:00:34,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834476 2024-11-14T14:00:34,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834476 2024-11-14T14:00:34,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834476) roll requested 2024-11-14T14:00:34,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834483 2024-11-14T14:00:34,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834483 2024-11-14T14:00:34,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834483) roll requested 2024-11-14T14:00:34,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834493 2024-11-14T14:00:34,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834493 2024-11-14T14:00:34,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834493) roll requested 2024-11-14T14:00:34,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834502 2024-11-14T14:00:34,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834502 2024-11-14T14:00:34,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834502) roll requested 2024-11-14T14:00:34,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834515 2024-11-14T14:00:34,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834515 2024-11-14T14:00:34,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834515) roll requested 2024-11-14T14:00:34,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834522 2024-11-14T14:00:34,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834522 2024-11-14T14:00:34,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834522) roll requested 2024-11-14T14:00:34,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834533 2024-11-14T14:00:34,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834533 2024-11-14T14:00:34,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834533) roll requested 2024-11-14T14:00:34,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834543 2024-11-14T14:00:34,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834543 2024-11-14T14:00:34,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834543) roll requested 2024-11-14T14:00:34,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834555 2024-11-14T14:00:34,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834555 2024-11-14T14:00:34,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834555) roll requested 2024-11-14T14:00:34,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834566 2024-11-14T14:00:34,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834566 2024-11-14T14:00:34,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834566) roll requested 2024-11-14T14:00:34,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834578 2024-11-14T14:00:34,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834578 2024-11-14T14:00:34,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834578) roll requested 2024-11-14T14:00:34,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834589 2024-11-14T14:00:34,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834589 2024-11-14T14:00:34,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834589) roll requested 2024-11-14T14:00:34,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834596 2024-11-14T14:00:34,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834596 2024-11-14T14:00:34,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834596) roll requested 2024-11-14T14:00:34,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834605 2024-11-14T14:00:34,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834605 2024-11-14T14:00:34,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834605) roll requested 2024-11-14T14:00:34,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834612 2024-11-14T14:00:34,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834612 2024-11-14T14:00:34,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834612) roll requested 2024-11-14T14:00:34,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834621 2024-11-14T14:00:34,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834621 2024-11-14T14:00:34,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834621) roll requested 2024-11-14T14:00:34,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834628 2024-11-14T14:00:34,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834628 2024-11-14T14:00:34,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834628) roll requested 2024-11-14T14:00:34,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834635 2024-11-14T14:00:34,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834635 2024-11-14T14:00:34,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834635) roll requested 2024-11-14T14:00:34,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834641 2024-11-14T14:00:34,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834641 2024-11-14T14:00:34,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834641) roll requested 2024-11-14T14:00:34,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834649 2024-11-14T14:00:34,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834649 2024-11-14T14:00:34,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834649) roll requested 2024-11-14T14:00:34,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834656 2024-11-14T14:00:34,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834656 2024-11-14T14:00:34,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834656) roll requested 2024-11-14T14:00:34,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834662 2024-11-14T14:00:34,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834662 2024-11-14T14:00:34,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834662) roll requested 2024-11-14T14:00:34,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834669 2024-11-14T14:00:34,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834669 2024-11-14T14:00:34,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834669) roll requested 2024-11-14T14:00:34,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834676 2024-11-14T14:00:34,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834676 2024-11-14T14:00:34,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834676) roll requested 2024-11-14T14:00:34,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834682 2024-11-14T14:00:34,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834682 2024-11-14T14:00:34,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834682) roll requested 2024-11-14T14:00:34,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834689 2024-11-14T14:00:34,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834689 2024-11-14T14:00:34,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834689) roll requested 2024-11-14T14:00:34,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834696 2024-11-14T14:00:34,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834696 2024-11-14T14:00:34,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834696) roll requested 2024-11-14T14:00:34,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834702 2024-11-14T14:00:34,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834702 2024-11-14T14:00:34,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834702) roll requested 2024-11-14T14:00:34,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834709 2024-11-14T14:00:34,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834709 2024-11-14T14:00:34,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834709) roll requested 2024-11-14T14:00:34,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834715 2024-11-14T14:00:34,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834715 2024-11-14T14:00:34,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834715) roll requested 2024-11-14T14:00:34,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834722 2024-11-14T14:00:34,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834722 2024-11-14T14:00:34,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834722) roll requested 2024-11-14T14:00:34,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834729 2024-11-14T14:00:34,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834729 2024-11-14T14:00:34,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834729) roll requested 2024-11-14T14:00:34,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834735 2024-11-14T14:00:34,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834735 2024-11-14T14:00:34,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834735) roll requested 2024-11-14T14:00:34,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834742 2024-11-14T14:00:34,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834742 2024-11-14T14:00:34,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834742) roll requested 2024-11-14T14:00:34,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834751 2024-11-14T14:00:34,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834751 2024-11-14T14:00:34,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834751) roll requested 2024-11-14T14:00:34,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834758 2024-11-14T14:00:34,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834758 2024-11-14T14:00:34,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834758) roll requested 2024-11-14T14:00:34,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834764 2024-11-14T14:00:34,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834764 2024-11-14T14:00:34,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834764) roll requested 2024-11-14T14:00:34,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834771 2024-11-14T14:00:34,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834771 2024-11-14T14:00:34,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834771) roll requested 2024-11-14T14:00:34,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834777 2024-11-14T14:00:34,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834777 2024-11-14T14:00:34,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834777) roll requested 2024-11-14T14:00:34,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834784 2024-11-14T14:00:34,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834784 2024-11-14T14:00:34,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834784) roll requested 2024-11-14T14:00:34,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834790 2024-11-14T14:00:34,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834790 2024-11-14T14:00:34,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834790) roll requested 2024-11-14T14:00:34,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834798 2024-11-14T14:00:34,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834798 2024-11-14T14:00:34,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834798) roll requested 2024-11-14T14:00:34,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834804 2024-11-14T14:00:34,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834804 2024-11-14T14:00:34,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834804) roll requested 2024-11-14T14:00:34,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834811 2024-11-14T14:00:34,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834811 2024-11-14T14:00:34,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834811) roll requested 2024-11-14T14:00:34,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834817 2024-11-14T14:00:34,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834817 2024-11-14T14:00:34,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834817) roll requested 2024-11-14T14:00:34,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834825 2024-11-14T14:00:34,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834825 2024-11-14T14:00:34,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834825) roll requested 2024-11-14T14:00:34,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834831 2024-11-14T14:00:34,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834831 2024-11-14T14:00:34,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834831) roll requested 2024-11-14T14:00:34,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834838 2024-11-14T14:00:34,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834838 2024-11-14T14:00:34,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834838) roll requested 2024-11-14T14:00:34,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834844 2024-11-14T14:00:34,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834844 2024-11-14T14:00:34,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834844) roll requested 2024-11-14T14:00:34,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834851 2024-11-14T14:00:34,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834851 2024-11-14T14:00:34,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834851) roll requested 2024-11-14T14:00:34,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834859 2024-11-14T14:00:34,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834859 2024-11-14T14:00:34,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834859) roll requested 2024-11-14T14:00:34,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834866 2024-11-14T14:00:34,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834866 2024-11-14T14:00:34,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834866) roll requested 2024-11-14T14:00:34,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834873 2024-11-14T14:00:34,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834873 2024-11-14T14:00:34,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834873) roll requested 2024-11-14T14:00:34,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834880 2024-11-14T14:00:34,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834880 2024-11-14T14:00:34,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834880) roll requested 2024-11-14T14:00:34,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834890 2024-11-14T14:00:34,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834890 2024-11-14T14:00:34,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834890) roll requested 2024-11-14T14:00:34,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834897 2024-11-14T14:00:34,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834897 2024-11-14T14:00:34,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834897) roll requested 2024-11-14T14:00:34,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834904 2024-11-14T14:00:34,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834904 2024-11-14T14:00:34,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834904) roll requested 2024-11-14T14:00:34,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834918 2024-11-14T14:00:34,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834918 2024-11-14T14:00:34,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834918) roll requested 2024-11-14T14:00:34,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834925 2024-11-14T14:00:34,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834925 2024-11-14T14:00:34,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834925) roll requested 2024-11-14T14:00:34,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834932 2024-11-14T14:00:34,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834932 2024-11-14T14:00:34,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834932) roll requested 2024-11-14T14:00:34,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834938 2024-11-14T14:00:34,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834938 2024-11-14T14:00:34,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834938) roll requested 2024-11-14T14:00:34,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834946 2024-11-14T14:00:34,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834946 2024-11-14T14:00:34,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834946) roll requested 2024-11-14T14:00:34,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834952 2024-11-14T14:00:34,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834952 2024-11-14T14:00:34,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834952) roll requested 2024-11-14T14:00:34,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834959 2024-11-14T14:00:34,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834959 2024-11-14T14:00:34,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834959) roll requested 2024-11-14T14:00:34,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834966 2024-11-14T14:00:34,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834966 2024-11-14T14:00:34,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834966) roll requested 2024-11-14T14:00:34,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834972 2024-11-14T14:00:34,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834972 2024-11-14T14:00:34,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834972) roll requested 2024-11-14T14:00:34,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834979 2024-11-14T14:00:34,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834979 2024-11-14T14:00:34,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834979) roll requested 2024-11-14T14:00:34,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834990 2024-11-14T14:00:34,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:34,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834990 2024-11-14T14:00:34,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:34,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:34,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834990) roll requested 2024-11-14T14:00:34,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592834996 2024-11-14T14:00:35,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834996 2024-11-14T14:00:35,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592834996) roll requested 2024-11-14T14:00:35,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835003 2024-11-14T14:00:35,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592834996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835003 2024-11-14T14:00:35,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835003) roll requested 2024-11-14T14:00:35,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835010 2024-11-14T14:00:35,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835010 2024-11-14T14:00:35,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835010) roll requested 2024-11-14T14:00:35,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835017 2024-11-14T14:00:35,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835017 2024-11-14T14:00:35,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835017) roll requested 2024-11-14T14:00:35,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835023 2024-11-14T14:00:35,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835023 2024-11-14T14:00:35,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835023) roll requested 2024-11-14T14:00:35,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835030 2024-11-14T14:00:35,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835030 2024-11-14T14:00:35,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835030) roll requested 2024-11-14T14:00:35,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835037 2024-11-14T14:00:35,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835037 2024-11-14T14:00:35,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835037) roll requested 2024-11-14T14:00:35,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835043 2024-11-14T14:00:35,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835043 2024-11-14T14:00:35,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835043) roll requested 2024-11-14T14:00:35,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835050 2024-11-14T14:00:35,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835050 2024-11-14T14:00:35,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835050) roll requested 2024-11-14T14:00:35,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835056 2024-11-14T14:00:35,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835056 2024-11-14T14:00:35,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835056) roll requested 2024-11-14T14:00:35,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835064 2024-11-14T14:00:35,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835064 2024-11-14T14:00:35,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835064) roll requested 2024-11-14T14:00:35,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835070 2024-11-14T14:00:35,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835070 2024-11-14T14:00:35,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835070) roll requested 2024-11-14T14:00:35,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835077 2024-11-14T14:00:35,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835077 2024-11-14T14:00:35,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835077) roll requested 2024-11-14T14:00:35,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835084 2024-11-14T14:00:35,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835084 2024-11-14T14:00:35,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835084) roll requested 2024-11-14T14:00:35,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835090 2024-11-14T14:00:35,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835090 2024-11-14T14:00:35,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835090) roll requested 2024-11-14T14:00:35,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835097 2024-11-14T14:00:35,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835097 2024-11-14T14:00:35,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835097) roll requested 2024-11-14T14:00:35,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835104 2024-11-14T14:00:35,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835104 2024-11-14T14:00:35,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835104) roll requested 2024-11-14T14:00:35,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835114 2024-11-14T14:00:35,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835114 2024-11-14T14:00:35,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835114) roll requested 2024-11-14T14:00:35,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835121 2024-11-14T14:00:35,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835121 2024-11-14T14:00:35,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835121) roll requested 2024-11-14T14:00:35,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835128 2024-11-14T14:00:35,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835128 2024-11-14T14:00:35,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835128) roll requested 2024-11-14T14:00:35,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835135 2024-11-14T14:00:35,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835135 2024-11-14T14:00:35,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835135) roll requested 2024-11-14T14:00:35,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835147 2024-11-14T14:00:35,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835147 2024-11-14T14:00:35,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835147) roll requested 2024-11-14T14:00:35,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835157 2024-11-14T14:00:35,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835157 2024-11-14T14:00:35,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835157) roll requested 2024-11-14T14:00:35,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835165 2024-11-14T14:00:35,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835165 2024-11-14T14:00:35,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835165) roll requested 2024-11-14T14:00:35,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835175 2024-11-14T14:00:35,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835175 2024-11-14T14:00:35,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835175) roll requested 2024-11-14T14:00:35,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835183 2024-11-14T14:00:35,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835183 2024-11-14T14:00:35,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835183) roll requested 2024-11-14T14:00:35,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835189 2024-11-14T14:00:35,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835189 2024-11-14T14:00:35,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835189) roll requested 2024-11-14T14:00:35,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835196 2024-11-14T14:00:35,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835196 2024-11-14T14:00:35,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835196) roll requested 2024-11-14T14:00:35,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835203 2024-11-14T14:00:35,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835203 2024-11-14T14:00:35,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835203) roll requested 2024-11-14T14:00:35,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835214 2024-11-14T14:00:35,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835214 2024-11-14T14:00:35,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835214) roll requested 2024-11-14T14:00:35,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835221 2024-11-14T14:00:35,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835221 2024-11-14T14:00:35,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835221) roll requested 2024-11-14T14:00:35,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835229 2024-11-14T14:00:35,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835229 2024-11-14T14:00:35,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835229) roll requested 2024-11-14T14:00:35,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835235 2024-11-14T14:00:35,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835235 2024-11-14T14:00:35,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835235) roll requested 2024-11-14T14:00:35,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835243 2024-11-14T14:00:35,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835243 2024-11-14T14:00:35,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835243) roll requested 2024-11-14T14:00:35,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835250 2024-11-14T14:00:35,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835250 2024-11-14T14:00:35,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835250) roll requested 2024-11-14T14:00:35,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835262 2024-11-14T14:00:35,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835262 2024-11-14T14:00:35,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835262) roll requested 2024-11-14T14:00:35,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835272 2024-11-14T14:00:35,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835272 2024-11-14T14:00:35,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835272) roll requested 2024-11-14T14:00:35,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835279 2024-11-14T14:00:35,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835279 2024-11-14T14:00:35,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835279) roll requested 2024-11-14T14:00:35,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835286 2024-11-14T14:00:35,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835286 2024-11-14T14:00:35,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835286) roll requested 2024-11-14T14:00:35,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835294 2024-11-14T14:00:35,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835294 2024-11-14T14:00:35,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835294) roll requested 2024-11-14T14:00:35,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835302 2024-11-14T14:00:35,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835302 2024-11-14T14:00:35,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835302) roll requested 2024-11-14T14:00:35,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835309 2024-11-14T14:00:35,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835309 2024-11-14T14:00:35,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835309) roll requested 2024-11-14T14:00:35,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835317 2024-11-14T14:00:35,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835317 2024-11-14T14:00:35,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835317) roll requested 2024-11-14T14:00:35,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835325 2024-11-14T14:00:35,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835325 2024-11-14T14:00:35,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835325) roll requested 2024-11-14T14:00:35,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835333 2024-11-14T14:00:35,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835333 2024-11-14T14:00:35,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835333) roll requested 2024-11-14T14:00:35,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835341 2024-11-14T14:00:35,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835341 2024-11-14T14:00:35,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835341) roll requested 2024-11-14T14:00:35,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835348 2024-11-14T14:00:35,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835348 2024-11-14T14:00:35,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835348) roll requested 2024-11-14T14:00:35,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835355 2024-11-14T14:00:35,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835355 2024-11-14T14:00:35,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835355) roll requested 2024-11-14T14:00:35,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835362 2024-11-14T14:00:35,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835362 2024-11-14T14:00:35,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835362) roll requested 2024-11-14T14:00:35,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835368 2024-11-14T14:00:35,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835368 2024-11-14T14:00:35,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835368) roll requested 2024-11-14T14:00:35,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835374 2024-11-14T14:00:35,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835374 2024-11-14T14:00:35,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835374) roll requested 2024-11-14T14:00:35,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835385 2024-11-14T14:00:35,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835385 2024-11-14T14:00:35,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835385) roll requested 2024-11-14T14:00:35,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835391 2024-11-14T14:00:35,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835391 2024-11-14T14:00:35,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835391) roll requested 2024-11-14T14:00:35,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835398 2024-11-14T14:00:35,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835398 2024-11-14T14:00:35,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835398) roll requested 2024-11-14T14:00:35,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835405 2024-11-14T14:00:35,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835405 2024-11-14T14:00:35,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835405) roll requested 2024-11-14T14:00:35,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835411 2024-11-14T14:00:35,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835411 2024-11-14T14:00:35,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835411) roll requested 2024-11-14T14:00:35,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835418 2024-11-14T14:00:35,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835418 2024-11-14T14:00:35,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835418) roll requested 2024-11-14T14:00:35,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835425 2024-11-14T14:00:35,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835425 2024-11-14T14:00:35,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835425) roll requested 2024-11-14T14:00:35,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835432 2024-11-14T14:00:35,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835432 2024-11-14T14:00:35,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835432) roll requested 2024-11-14T14:00:35,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835439 2024-11-14T14:00:35,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835439 2024-11-14T14:00:35,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835439) roll requested 2024-11-14T14:00:35,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835445 2024-11-14T14:00:35,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835445 2024-11-14T14:00:35,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835445) roll requested 2024-11-14T14:00:35,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835455 2024-11-14T14:00:35,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835455 2024-11-14T14:00:35,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835455) roll requested 2024-11-14T14:00:35,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835461 2024-11-14T14:00:35,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835461 2024-11-14T14:00:35,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835461) roll requested 2024-11-14T14:00:35,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835473 2024-11-14T14:00:35,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835473 2024-11-14T14:00:35,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835473) roll requested 2024-11-14T14:00:35,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835479 2024-11-14T14:00:35,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835479 2024-11-14T14:00:35,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835479) roll requested 2024-11-14T14:00:35,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835486 2024-11-14T14:00:35,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835486 2024-11-14T14:00:35,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835486) roll requested 2024-11-14T14:00:35,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835493 2024-11-14T14:00:35,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835493 2024-11-14T14:00:35,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835493) roll requested 2024-11-14T14:00:35,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835500 2024-11-14T14:00:35,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835500 2024-11-14T14:00:35,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835500) roll requested 2024-11-14T14:00:35,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835509 2024-11-14T14:00:35,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835509 2024-11-14T14:00:35,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835509) roll requested 2024-11-14T14:00:35,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835515 2024-11-14T14:00:35,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835515 2024-11-14T14:00:35,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835515) roll requested 2024-11-14T14:00:35,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835521 2024-11-14T14:00:35,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835521 2024-11-14T14:00:35,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835521) roll requested 2024-11-14T14:00:35,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835528 2024-11-14T14:00:35,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835528 2024-11-14T14:00:35,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835528) roll requested 2024-11-14T14:00:35,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835535 2024-11-14T14:00:35,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835535 2024-11-14T14:00:35,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835535) roll requested 2024-11-14T14:00:35,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835543 2024-11-14T14:00:35,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835543 2024-11-14T14:00:35,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835543) roll requested 2024-11-14T14:00:35,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835551 2024-11-14T14:00:35,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835551 2024-11-14T14:00:35,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835551) roll requested 2024-11-14T14:00:35,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835557 2024-11-14T14:00:35,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835557 2024-11-14T14:00:35,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835557) roll requested 2024-11-14T14:00:35,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835565 2024-11-14T14:00:35,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835565 2024-11-14T14:00:35,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835565) roll requested 2024-11-14T14:00:35,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835573 2024-11-14T14:00:35,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835573 2024-11-14T14:00:35,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835573) roll requested 2024-11-14T14:00:35,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835580 2024-11-14T14:00:35,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835580 2024-11-14T14:00:35,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835580) roll requested 2024-11-14T14:00:35,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835586 2024-11-14T14:00:35,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835586 2024-11-14T14:00:35,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835586) roll requested 2024-11-14T14:00:35,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835593 2024-11-14T14:00:35,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835593 2024-11-14T14:00:35,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835593) roll requested 2024-11-14T14:00:35,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835599 2024-11-14T14:00:35,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835599 2024-11-14T14:00:35,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835599) roll requested 2024-11-14T14:00:35,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835606 2024-11-14T14:00:35,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835606 2024-11-14T14:00:35,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835606) roll requested 2024-11-14T14:00:35,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835613 2024-11-14T14:00:35,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835613 2024-11-14T14:00:35,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835613) roll requested 2024-11-14T14:00:35,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835619 2024-11-14T14:00:35,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835619 2024-11-14T14:00:35,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835619) roll requested 2024-11-14T14:00:35,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835626 2024-11-14T14:00:35,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835626 2024-11-14T14:00:35,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835626) roll requested 2024-11-14T14:00:35,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835632 2024-11-14T14:00:35,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835632 2024-11-14T14:00:35,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835632) roll requested 2024-11-14T14:00:35,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835638 2024-11-14T14:00:35,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835638 2024-11-14T14:00:35,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835638) roll requested 2024-11-14T14:00:35,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835645 2024-11-14T14:00:35,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835645 2024-11-14T14:00:35,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835645) roll requested 2024-11-14T14:00:35,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835652 2024-11-14T14:00:35,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835652 2024-11-14T14:00:35,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835652) roll requested 2024-11-14T14:00:35,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835659 2024-11-14T14:00:35,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835659 2024-11-14T14:00:35,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835659) roll requested 2024-11-14T14:00:35,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835665 2024-11-14T14:00:35,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835665 2024-11-14T14:00:35,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835665) roll requested 2024-11-14T14:00:35,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835672 2024-11-14T14:00:35,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835672 2024-11-14T14:00:35,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835672) roll requested 2024-11-14T14:00:35,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835678 2024-11-14T14:00:35,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835678 2024-11-14T14:00:35,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835678) roll requested 2024-11-14T14:00:35,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835684 2024-11-14T14:00:35,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835684 2024-11-14T14:00:35,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835684) roll requested 2024-11-14T14:00:35,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835691 2024-11-14T14:00:35,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835691 2024-11-14T14:00:35,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835691) roll requested 2024-11-14T14:00:35,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835697 2024-11-14T14:00:35,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835697 2024-11-14T14:00:35,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835697) roll requested 2024-11-14T14:00:35,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835704 2024-11-14T14:00:35,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835704 2024-11-14T14:00:35,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835704) roll requested 2024-11-14T14:00:35,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835711 2024-11-14T14:00:35,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835711 2024-11-14T14:00:35,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835711) roll requested 2024-11-14T14:00:35,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835718 2024-11-14T14:00:35,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835718 2024-11-14T14:00:35,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835718) roll requested 2024-11-14T14:00:35,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835724 2024-11-14T14:00:35,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835724 2024-11-14T14:00:35,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835724) roll requested 2024-11-14T14:00:35,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835731 2024-11-14T14:00:35,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835731 2024-11-14T14:00:35,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835731) roll requested 2024-11-14T14:00:35,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835737 2024-11-14T14:00:35,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835737 2024-11-14T14:00:35,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835737) roll requested 2024-11-14T14:00:35,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835744 2024-11-14T14:00:35,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835744 2024-11-14T14:00:35,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835744) roll requested 2024-11-14T14:00:35,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835751 2024-11-14T14:00:35,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835751 2024-11-14T14:00:35,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835751) roll requested 2024-11-14T14:00:35,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835758 2024-11-14T14:00:35,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835758 2024-11-14T14:00:35,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835758) roll requested 2024-11-14T14:00:35,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835765 2024-11-14T14:00:35,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835765 2024-11-14T14:00:35,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835765) roll requested 2024-11-14T14:00:35,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835771 2024-11-14T14:00:35,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835771 2024-11-14T14:00:35,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835771) roll requested 2024-11-14T14:00:35,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835778 2024-11-14T14:00:35,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835778 2024-11-14T14:00:35,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835778) roll requested 2024-11-14T14:00:35,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835785 2024-11-14T14:00:35,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835785 2024-11-14T14:00:35,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835785) roll requested 2024-11-14T14:00:35,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835792 2024-11-14T14:00:35,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835792 2024-11-14T14:00:35,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835792) roll requested 2024-11-14T14:00:35,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835800 2024-11-14T14:00:35,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835800 2024-11-14T14:00:35,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835800) roll requested 2024-11-14T14:00:35,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835807 2024-11-14T14:00:35,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835807 2024-11-14T14:00:35,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835807) roll requested 2024-11-14T14:00:35,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835813 2024-11-14T14:00:35,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835813 2024-11-14T14:00:35,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835813) roll requested 2024-11-14T14:00:35,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835819 2024-11-14T14:00:35,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835819 2024-11-14T14:00:35,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835819) roll requested 2024-11-14T14:00:35,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835826 2024-11-14T14:00:35,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835826 2024-11-14T14:00:35,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835826) roll requested 2024-11-14T14:00:35,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835832 2024-11-14T14:00:35,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835832 2024-11-14T14:00:35,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835832) roll requested 2024-11-14T14:00:35,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835839 2024-11-14T14:00:35,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835839 2024-11-14T14:00:35,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835839) roll requested 2024-11-14T14:00:35,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835845 2024-11-14T14:00:35,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835845 2024-11-14T14:00:35,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835845) roll requested 2024-11-14T14:00:35,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835852 2024-11-14T14:00:35,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835852 2024-11-14T14:00:35,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835852) roll requested 2024-11-14T14:00:35,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835859 2024-11-14T14:00:35,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835859 2024-11-14T14:00:35,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835859) roll requested 2024-11-14T14:00:35,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835870 2024-11-14T14:00:35,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835870 2024-11-14T14:00:35,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835870) roll requested 2024-11-14T14:00:35,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835876 2024-11-14T14:00:35,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835876 2024-11-14T14:00:35,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835876) roll requested 2024-11-14T14:00:35,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835883 2024-11-14T14:00:35,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835883 2024-11-14T14:00:35,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835883) roll requested 2024-11-14T14:00:35,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835889 2024-11-14T14:00:35,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835889 2024-11-14T14:00:35,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835889) roll requested 2024-11-14T14:00:35,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835896 2024-11-14T14:00:35,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835896 2024-11-14T14:00:35,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835896) roll requested 2024-11-14T14:00:35,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835903 2024-11-14T14:00:35,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835903 2024-11-14T14:00:35,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835903) roll requested 2024-11-14T14:00:35,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835909 2024-11-14T14:00:35,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835909 2024-11-14T14:00:35,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835909) roll requested 2024-11-14T14:00:35,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835915 2024-11-14T14:00:35,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835915 2024-11-14T14:00:35,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835915) roll requested 2024-11-14T14:00:35,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835922 2024-11-14T14:00:35,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835922 2024-11-14T14:00:35,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835922) roll requested 2024-11-14T14:00:35,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835928 2024-11-14T14:00:35,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835928 2024-11-14T14:00:35,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835928) roll requested 2024-11-14T14:00:35,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835934 2024-11-14T14:00:35,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835934 2024-11-14T14:00:35,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835934) roll requested 2024-11-14T14:00:35,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835941 2024-11-14T14:00:35,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835941 2024-11-14T14:00:35,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835941) roll requested 2024-11-14T14:00:35,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835949 2024-11-14T14:00:35,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835949 2024-11-14T14:00:35,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835949) roll requested 2024-11-14T14:00:35,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835957 2024-11-14T14:00:35,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835957 2024-11-14T14:00:35,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835957) roll requested 2024-11-14T14:00:35,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835964 2024-11-14T14:00:35,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835964 2024-11-14T14:00:35,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835964) roll requested 2024-11-14T14:00:35,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835971 2024-11-14T14:00:35,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835971 2024-11-14T14:00:35,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835971) roll requested 2024-11-14T14:00:35,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835977 2024-11-14T14:00:35,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835977 2024-11-14T14:00:35,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835977) roll requested 2024-11-14T14:00:35,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835983 2024-11-14T14:00:35,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835983 2024-11-14T14:00:35,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835983) roll requested 2024-11-14T14:00:35,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835990 2024-11-14T14:00:35,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:35,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835990 2024-11-14T14:00:35,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:35,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:35,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835990) roll requested 2024-11-14T14:00:35,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592835996 2024-11-14T14:00:36,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835996 2024-11-14T14:00:36,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592835996) roll requested 2024-11-14T14:00:36,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836002 2024-11-14T14:00:36,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592835996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836002 2024-11-14T14:00:36,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836002) roll requested 2024-11-14T14:00:36,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836009 2024-11-14T14:00:36,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836009 2024-11-14T14:00:36,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836009) roll requested 2024-11-14T14:00:36,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836017 2024-11-14T14:00:36,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836017 2024-11-14T14:00:36,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836017) roll requested 2024-11-14T14:00:36,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836024 2024-11-14T14:00:36,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836024 2024-11-14T14:00:36,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836024) roll requested 2024-11-14T14:00:36,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836030 2024-11-14T14:00:36,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836030 2024-11-14T14:00:36,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836030) roll requested 2024-11-14T14:00:36,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836037 2024-11-14T14:00:36,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836037 2024-11-14T14:00:36,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836037) roll requested 2024-11-14T14:00:36,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836043 2024-11-14T14:00:36,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836043 2024-11-14T14:00:36,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836043) roll requested 2024-11-14T14:00:36,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836050 2024-11-14T14:00:36,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836050 2024-11-14T14:00:36,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836050) roll requested 2024-11-14T14:00:36,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836056 2024-11-14T14:00:36,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836056 2024-11-14T14:00:36,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836056) roll requested 2024-11-14T14:00:36,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836063 2024-11-14T14:00:36,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836063 2024-11-14T14:00:36,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836063) roll requested 2024-11-14T14:00:36,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836069 2024-11-14T14:00:36,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836069 2024-11-14T14:00:36,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836069) roll requested 2024-11-14T14:00:36,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836076 2024-11-14T14:00:36,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836076 2024-11-14T14:00:36,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836076) roll requested 2024-11-14T14:00:36,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836082 2024-11-14T14:00:36,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836082 2024-11-14T14:00:36,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836082) roll requested 2024-11-14T14:00:36,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836088 2024-11-14T14:00:36,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836088 2024-11-14T14:00:36,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836088) roll requested 2024-11-14T14:00:36,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836094 2024-11-14T14:00:36,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836094 2024-11-14T14:00:36,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836094) roll requested 2024-11-14T14:00:36,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836101 2024-11-14T14:00:36,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836101 2024-11-14T14:00:36,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836101) roll requested 2024-11-14T14:00:36,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836107 2024-11-14T14:00:36,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836107 2024-11-14T14:00:36,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836107) roll requested 2024-11-14T14:00:36,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836114 2024-11-14T14:00:36,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836114 2024-11-14T14:00:36,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836114) roll requested 2024-11-14T14:00:36,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836121 2024-11-14T14:00:36,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836121 2024-11-14T14:00:36,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836121) roll requested 2024-11-14T14:00:36,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836130 2024-11-14T14:00:36,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836130 2024-11-14T14:00:36,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836130) roll requested 2024-11-14T14:00:36,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836136 2024-11-14T14:00:36,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836136 2024-11-14T14:00:36,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836136) roll requested 2024-11-14T14:00:36,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836143 2024-11-14T14:00:36,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836143 2024-11-14T14:00:36,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836143) roll requested 2024-11-14T14:00:36,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836150 2024-11-14T14:00:36,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836150 2024-11-14T14:00:36,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836150) roll requested 2024-11-14T14:00:36,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836157 2024-11-14T14:00:36,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836157 2024-11-14T14:00:36,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836157) roll requested 2024-11-14T14:00:36,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836163 2024-11-14T14:00:36,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836163 2024-11-14T14:00:36,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836163) roll requested 2024-11-14T14:00:36,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836170 2024-11-14T14:00:36,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836170 2024-11-14T14:00:36,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836170) roll requested 2024-11-14T14:00:36,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836176 2024-11-14T14:00:36,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836176 2024-11-14T14:00:36,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836176) roll requested 2024-11-14T14:00:36,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836182 2024-11-14T14:00:36,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836182 2024-11-14T14:00:36,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836182) roll requested 2024-11-14T14:00:36,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836189 2024-11-14T14:00:36,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836189 2024-11-14T14:00:36,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836189) roll requested 2024-11-14T14:00:36,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836195 2024-11-14T14:00:36,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836195 2024-11-14T14:00:36,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836195) roll requested 2024-11-14T14:00:36,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836202 2024-11-14T14:00:36,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836202 2024-11-14T14:00:36,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836202) roll requested 2024-11-14T14:00:36,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836212 2024-11-14T14:00:36,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836212 2024-11-14T14:00:36,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836212) roll requested 2024-11-14T14:00:36,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836220 2024-11-14T14:00:36,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836220 2024-11-14T14:00:36,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836220) roll requested 2024-11-14T14:00:36,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836226 2024-11-14T14:00:36,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836226 2024-11-14T14:00:36,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836226) roll requested 2024-11-14T14:00:36,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836233 2024-11-14T14:00:36,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836233 2024-11-14T14:00:36,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836233) roll requested 2024-11-14T14:00:36,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836239 2024-11-14T14:00:36,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836239 2024-11-14T14:00:36,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836239) roll requested 2024-11-14T14:00:36,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836246 2024-11-14T14:00:36,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836246 2024-11-14T14:00:36,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836246) roll requested 2024-11-14T14:00:36,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836252 2024-11-14T14:00:36,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836252 2024-11-14T14:00:36,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836252) roll requested 2024-11-14T14:00:36,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836258 2024-11-14T14:00:36,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836258 2024-11-14T14:00:36,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836258) roll requested 2024-11-14T14:00:36,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836265 2024-11-14T14:00:36,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836265 2024-11-14T14:00:36,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836265) roll requested 2024-11-14T14:00:36,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836271 2024-11-14T14:00:36,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836271 2024-11-14T14:00:36,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836271) roll requested 2024-11-14T14:00:36,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836278 2024-11-14T14:00:36,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836278 2024-11-14T14:00:36,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836278) roll requested 2024-11-14T14:00:36,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836285 2024-11-14T14:00:36,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836285 2024-11-14T14:00:36,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836285) roll requested 2024-11-14T14:00:36,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836291 2024-11-14T14:00:36,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836291 2024-11-14T14:00:36,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836291) roll requested 2024-11-14T14:00:36,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836298 2024-11-14T14:00:36,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836298 2024-11-14T14:00:36,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836298) roll requested 2024-11-14T14:00:36,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836304 2024-11-14T14:00:36,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836304 2024-11-14T14:00:36,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836304) roll requested 2024-11-14T14:00:36,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836311 2024-11-14T14:00:36,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836311 2024-11-14T14:00:36,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836311) roll requested 2024-11-14T14:00:36,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836317 2024-11-14T14:00:36,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836317 2024-11-14T14:00:36,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836317) roll requested 2024-11-14T14:00:36,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836324 2024-11-14T14:00:36,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836324 2024-11-14T14:00:36,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836324) roll requested 2024-11-14T14:00:36,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836332 2024-11-14T14:00:36,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836332 2024-11-14T14:00:36,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836332) roll requested 2024-11-14T14:00:36,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836339 2024-11-14T14:00:36,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836339 2024-11-14T14:00:36,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836339) roll requested 2024-11-14T14:00:36,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836345 2024-11-14T14:00:36,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836345 2024-11-14T14:00:36,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836345) roll requested 2024-11-14T14:00:36,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836351 2024-11-14T14:00:36,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836351 2024-11-14T14:00:36,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836351) roll requested 2024-11-14T14:00:36,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836358 2024-11-14T14:00:36,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836358 2024-11-14T14:00:36,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836358) roll requested 2024-11-14T14:00:36,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836366 2024-11-14T14:00:36,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836366 2024-11-14T14:00:36,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836366) roll requested 2024-11-14T14:00:36,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836372 2024-11-14T14:00:36,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836372 2024-11-14T14:00:36,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836372) roll requested 2024-11-14T14:00:36,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836379 2024-11-14T14:00:36,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836379 2024-11-14T14:00:36,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836379) roll requested 2024-11-14T14:00:36,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836385 2024-11-14T14:00:36,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836385 2024-11-14T14:00:36,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836385) roll requested 2024-11-14T14:00:36,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836391 2024-11-14T14:00:36,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836391 2024-11-14T14:00:36,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836391) roll requested 2024-11-14T14:00:36,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836398 2024-11-14T14:00:36,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836398 2024-11-14T14:00:36,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836398) roll requested 2024-11-14T14:00:36,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836404 2024-11-14T14:00:36,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836404 2024-11-14T14:00:36,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836404) roll requested 2024-11-14T14:00:36,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836413 2024-11-14T14:00:36,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836413 2024-11-14T14:00:36,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836413) roll requested 2024-11-14T14:00:36,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836420 2024-11-14T14:00:36,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836420 2024-11-14T14:00:36,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836420) roll requested 2024-11-14T14:00:36,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836426 2024-11-14T14:00:36,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836426 2024-11-14T14:00:36,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836426) roll requested 2024-11-14T14:00:36,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836433 2024-11-14T14:00:36,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836433 2024-11-14T14:00:36,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836433) roll requested 2024-11-14T14:00:36,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836439 2024-11-14T14:00:36,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836439 2024-11-14T14:00:36,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836439) roll requested 2024-11-14T14:00:36,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836445 2024-11-14T14:00:36,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836445 2024-11-14T14:00:36,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836445) roll requested 2024-11-14T14:00:36,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836452 2024-11-14T14:00:36,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836452 2024-11-14T14:00:36,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836452) roll requested 2024-11-14T14:00:36,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836458 2024-11-14T14:00:36,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836458 2024-11-14T14:00:36,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836458) roll requested 2024-11-14T14:00:36,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836465 2024-11-14T14:00:36,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836465 2024-11-14T14:00:36,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836465) roll requested 2024-11-14T14:00:36,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836472 2024-11-14T14:00:36,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836472 2024-11-14T14:00:36,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836472) roll requested 2024-11-14T14:00:36,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836478 2024-11-14T14:00:36,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836478 2024-11-14T14:00:36,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836478) roll requested 2024-11-14T14:00:36,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836485 2024-11-14T14:00:36,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836485 2024-11-14T14:00:36,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836485) roll requested 2024-11-14T14:00:36,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836491 2024-11-14T14:00:36,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836491 2024-11-14T14:00:36,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836491) roll requested 2024-11-14T14:00:36,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836497 2024-11-14T14:00:36,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836497 2024-11-14T14:00:36,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836497) roll requested 2024-11-14T14:00:36,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836508 2024-11-14T14:00:36,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836508 2024-11-14T14:00:36,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836508) roll requested 2024-11-14T14:00:36,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836514 2024-11-14T14:00:36,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836514 2024-11-14T14:00:36,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836514) roll requested 2024-11-14T14:00:36,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836520 2024-11-14T14:00:36,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836520 2024-11-14T14:00:36,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836520) roll requested 2024-11-14T14:00:36,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836527 2024-11-14T14:00:36,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836527 2024-11-14T14:00:36,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836527) roll requested 2024-11-14T14:00:36,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836533 2024-11-14T14:00:36,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836533 2024-11-14T14:00:36,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836533) roll requested 2024-11-14T14:00:36,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836539 2024-11-14T14:00:36,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836539 2024-11-14T14:00:36,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836539) roll requested 2024-11-14T14:00:36,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836557 2024-11-14T14:00:36,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836557 2024-11-14T14:00:36,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836557) roll requested 2024-11-14T14:00:36,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836564 2024-11-14T14:00:36,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836564 2024-11-14T14:00:36,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836564) roll requested 2024-11-14T14:00:36,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836570 2024-11-14T14:00:36,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836570 2024-11-14T14:00:36,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836570) roll requested 2024-11-14T14:00:36,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836576 2024-11-14T14:00:36,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836576 2024-11-14T14:00:36,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836576) roll requested 2024-11-14T14:00:36,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836584 2024-11-14T14:00:36,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836584 2024-11-14T14:00:36,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836584) roll requested 2024-11-14T14:00:36,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836593 2024-11-14T14:00:36,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836593 2024-11-14T14:00:36,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836593) roll requested 2024-11-14T14:00:36,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836599 2024-11-14T14:00:36,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836599 2024-11-14T14:00:36,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836599) roll requested 2024-11-14T14:00:36,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836606 2024-11-14T14:00:36,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836606 2024-11-14T14:00:36,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836606) roll requested 2024-11-14T14:00:36,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836613 2024-11-14T14:00:36,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836613 2024-11-14T14:00:36,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836613) roll requested 2024-11-14T14:00:36,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836619 2024-11-14T14:00:36,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836619 2024-11-14T14:00:36,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836619) roll requested 2024-11-14T14:00:36,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836625 2024-11-14T14:00:36,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836625 2024-11-14T14:00:36,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836625) roll requested 2024-11-14T14:00:36,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836632 2024-11-14T14:00:36,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836632 2024-11-14T14:00:36,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836632) roll requested 2024-11-14T14:00:36,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836638 2024-11-14T14:00:36,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836638 2024-11-14T14:00:36,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836638) roll requested 2024-11-14T14:00:36,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836644 2024-11-14T14:00:36,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836644 2024-11-14T14:00:36,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836644) roll requested 2024-11-14T14:00:36,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836651 2024-11-14T14:00:36,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836651 2024-11-14T14:00:36,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836651) roll requested 2024-11-14T14:00:36,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836657 2024-11-14T14:00:36,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836657 2024-11-14T14:00:36,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836657) roll requested 2024-11-14T14:00:36,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836663 2024-11-14T14:00:36,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836663 2024-11-14T14:00:36,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836663) roll requested 2024-11-14T14:00:36,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836670 2024-11-14T14:00:36,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836670 2024-11-14T14:00:36,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836670) roll requested 2024-11-14T14:00:36,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836676 2024-11-14T14:00:36,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836676 2024-11-14T14:00:36,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836676) roll requested 2024-11-14T14:00:36,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836683 2024-11-14T14:00:36,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836683 2024-11-14T14:00:36,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836683) roll requested 2024-11-14T14:00:36,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836689 2024-11-14T14:00:36,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836689 2024-11-14T14:00:36,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836689) roll requested 2024-11-14T14:00:36,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836696 2024-11-14T14:00:36,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836696 2024-11-14T14:00:36,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836696) roll requested 2024-11-14T14:00:36,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836702 2024-11-14T14:00:36,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836702 2024-11-14T14:00:36,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836702) roll requested 2024-11-14T14:00:36,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836708 2024-11-14T14:00:36,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836708 2024-11-14T14:00:36,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836708) roll requested 2024-11-14T14:00:36,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836714 2024-11-14T14:00:36,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836714 2024-11-14T14:00:36,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836714) roll requested 2024-11-14T14:00:36,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836721 2024-11-14T14:00:36,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836721 2024-11-14T14:00:36,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836721) roll requested 2024-11-14T14:00:36,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836727 2024-11-14T14:00:36,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836727 2024-11-14T14:00:36,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836727) roll requested 2024-11-14T14:00:36,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836734 2024-11-14T14:00:36,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836734 2024-11-14T14:00:36,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836734) roll requested 2024-11-14T14:00:36,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836740 2024-11-14T14:00:36,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836740 2024-11-14T14:00:36,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836740) roll requested 2024-11-14T14:00:36,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836746 2024-11-14T14:00:36,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836746 2024-11-14T14:00:36,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836746) roll requested 2024-11-14T14:00:36,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836753 2024-11-14T14:00:36,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836753 2024-11-14T14:00:36,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836753) roll requested 2024-11-14T14:00:36,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836759 2024-11-14T14:00:36,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836759 2024-11-14T14:00:36,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836759) roll requested 2024-11-14T14:00:36,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836765 2024-11-14T14:00:36,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836765 2024-11-14T14:00:36,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836765) roll requested 2024-11-14T14:00:36,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836772 2024-11-14T14:00:36,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836772 2024-11-14T14:00:36,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836772) roll requested 2024-11-14T14:00:36,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836778 2024-11-14T14:00:36,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836778 2024-11-14T14:00:36,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836778) roll requested 2024-11-14T14:00:36,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836785 2024-11-14T14:00:36,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836785 2024-11-14T14:00:36,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836785) roll requested 2024-11-14T14:00:36,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836791 2024-11-14T14:00:36,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836791 2024-11-14T14:00:36,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836791) roll requested 2024-11-14T14:00:36,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836797 2024-11-14T14:00:36,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836797 2024-11-14T14:00:36,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836797) roll requested 2024-11-14T14:00:36,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836804 2024-11-14T14:00:36,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836804 2024-11-14T14:00:36,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836804) roll requested 2024-11-14T14:00:36,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836810 2024-11-14T14:00:36,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836810 2024-11-14T14:00:36,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836810) roll requested 2024-11-14T14:00:36,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836817 2024-11-14T14:00:36,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836817 2024-11-14T14:00:36,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836817) roll requested 2024-11-14T14:00:36,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836824 2024-11-14T14:00:36,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836824 2024-11-14T14:00:36,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836824) roll requested 2024-11-14T14:00:36,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836830 2024-11-14T14:00:36,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836830 2024-11-14T14:00:36,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836830) roll requested 2024-11-14T14:00:36,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836836 2024-11-14T14:00:36,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836836 2024-11-14T14:00:36,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836836) roll requested 2024-11-14T14:00:36,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836843 2024-11-14T14:00:36,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836843 2024-11-14T14:00:36,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836843) roll requested 2024-11-14T14:00:36,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836849 2024-11-14T14:00:36,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836849 2024-11-14T14:00:36,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836849) roll requested 2024-11-14T14:00:36,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836855 2024-11-14T14:00:36,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836855 2024-11-14T14:00:36,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836855) roll requested 2024-11-14T14:00:36,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836862 2024-11-14T14:00:36,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836862 2024-11-14T14:00:36,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836862) roll requested 2024-11-14T14:00:36,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836868 2024-11-14T14:00:36,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836868 2024-11-14T14:00:36,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836868) roll requested 2024-11-14T14:00:36,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836875 2024-11-14T14:00:36,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836875 2024-11-14T14:00:36,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836875) roll requested 2024-11-14T14:00:36,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836881 2024-11-14T14:00:36,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836881 2024-11-14T14:00:36,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836881) roll requested 2024-11-14T14:00:36,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836887 2024-11-14T14:00:36,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836887 2024-11-14T14:00:36,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836887) roll requested 2024-11-14T14:00:36,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836894 2024-11-14T14:00:36,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836894 2024-11-14T14:00:36,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836894) roll requested 2024-11-14T14:00:36,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836900 2024-11-14T14:00:36,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836900 2024-11-14T14:00:36,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836900) roll requested 2024-11-14T14:00:36,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836906 2024-11-14T14:00:36,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836906 2024-11-14T14:00:36,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836906) roll requested 2024-11-14T14:00:36,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836912 2024-11-14T14:00:36,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836912 2024-11-14T14:00:36,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836912) roll requested 2024-11-14T14:00:36,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836920 2024-11-14T14:00:36,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836920 2024-11-14T14:00:36,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836920) roll requested 2024-11-14T14:00:36,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836926 2024-11-14T14:00:36,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836926 2024-11-14T14:00:36,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836926) roll requested 2024-11-14T14:00:36,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836933 2024-11-14T14:00:36,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836933 2024-11-14T14:00:36,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836933) roll requested 2024-11-14T14:00:36,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836939 2024-11-14T14:00:36,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836939 2024-11-14T14:00:36,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836939) roll requested 2024-11-14T14:00:36,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836945 2024-11-14T14:00:36,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836945 2024-11-14T14:00:36,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836945) roll requested 2024-11-14T14:00:36,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836951 2024-11-14T14:00:36,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836951 2024-11-14T14:00:36,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836951) roll requested 2024-11-14T14:00:36,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836958 2024-11-14T14:00:36,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836958 2024-11-14T14:00:36,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836958) roll requested 2024-11-14T14:00:36,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836964 2024-11-14T14:00:36,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836964 2024-11-14T14:00:36,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836964) roll requested 2024-11-14T14:00:36,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836970 2024-11-14T14:00:36,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836970 2024-11-14T14:00:36,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836970) roll requested 2024-11-14T14:00:36,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836977 2024-11-14T14:00:36,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836977 2024-11-14T14:00:36,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836977) roll requested 2024-11-14T14:00:36,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836983 2024-11-14T14:00:36,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836983 2024-11-14T14:00:36,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836983) roll requested 2024-11-14T14:00:36,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836989 2024-11-14T14:00:36,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:36,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836989 2024-11-14T14:00:36,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:36,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:36,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836989) roll requested 2024-11-14T14:00:36,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592836997 2024-11-14T14:00:37,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836997 2024-11-14T14:00:37,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592836997) roll requested 2024-11-14T14:00:37,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837004 2024-11-14T14:00:37,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592836997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837004 2024-11-14T14:00:37,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837004) roll requested 2024-11-14T14:00:37,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837010 2024-11-14T14:00:37,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837010 2024-11-14T14:00:37,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837010) roll requested 2024-11-14T14:00:37,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837017 2024-11-14T14:00:37,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837017 2024-11-14T14:00:37,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837017) roll requested 2024-11-14T14:00:37,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837023 2024-11-14T14:00:37,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837023 2024-11-14T14:00:37,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837023) roll requested 2024-11-14T14:00:37,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837029 2024-11-14T14:00:37,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837029 2024-11-14T14:00:37,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837029) roll requested 2024-11-14T14:00:37,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837036 2024-11-14T14:00:37,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837036 2024-11-14T14:00:37,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837036) roll requested 2024-11-14T14:00:37,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837042 2024-11-14T14:00:37,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837042 2024-11-14T14:00:37,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837042) roll requested 2024-11-14T14:00:37,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837054 2024-11-14T14:00:37,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837054 2024-11-14T14:00:37,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837054) roll requested 2024-11-14T14:00:37,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837060 2024-11-14T14:00:37,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837060 2024-11-14T14:00:37,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837060) roll requested 2024-11-14T14:00:37,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837066 2024-11-14T14:00:37,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837066 2024-11-14T14:00:37,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837066) roll requested 2024-11-14T14:00:37,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837073 2024-11-14T14:00:37,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837073 2024-11-14T14:00:37,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837073) roll requested 2024-11-14T14:00:37,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837081 2024-11-14T14:00:37,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837081 2024-11-14T14:00:37,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837081) roll requested 2024-11-14T14:00:37,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837087 2024-11-14T14:00:37,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837087 2024-11-14T14:00:37,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837087) roll requested 2024-11-14T14:00:37,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837093 2024-11-14T14:00:37,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837093 2024-11-14T14:00:37,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837093) roll requested 2024-11-14T14:00:37,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837100 2024-11-14T14:00:37,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837100 2024-11-14T14:00:37,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837100) roll requested 2024-11-14T14:00:37,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837106 2024-11-14T14:00:37,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837106 2024-11-14T14:00:37,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837106) roll requested 2024-11-14T14:00:37,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837112 2024-11-14T14:00:37,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837112 2024-11-14T14:00:37,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837112) roll requested 2024-11-14T14:00:37,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837119 2024-11-14T14:00:37,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837119 2024-11-14T14:00:37,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837119) roll requested 2024-11-14T14:00:37,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837125 2024-11-14T14:00:37,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837125 2024-11-14T14:00:37,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837125) roll requested 2024-11-14T14:00:37,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837131 2024-11-14T14:00:37,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837131 2024-11-14T14:00:37,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837131) roll requested 2024-11-14T14:00:37,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837137 2024-11-14T14:00:37,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837137 2024-11-14T14:00:37,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837137) roll requested 2024-11-14T14:00:37,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837143 2024-11-14T14:00:37,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837143 2024-11-14T14:00:37,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837143) roll requested 2024-11-14T14:00:37,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837150 2024-11-14T14:00:37,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837150 2024-11-14T14:00:37,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837150) roll requested 2024-11-14T14:00:37,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837156 2024-11-14T14:00:37,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837156 2024-11-14T14:00:37,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837156) roll requested 2024-11-14T14:00:37,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837164 2024-11-14T14:00:37,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837164 2024-11-14T14:00:37,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837164) roll requested 2024-11-14T14:00:37,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837173 2024-11-14T14:00:37,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837173 2024-11-14T14:00:37,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837173) roll requested 2024-11-14T14:00:37,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837179 2024-11-14T14:00:37,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837179 2024-11-14T14:00:37,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837179) roll requested 2024-11-14T14:00:37,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837189 2024-11-14T14:00:37,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837189 2024-11-14T14:00:37,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837189) roll requested 2024-11-14T14:00:37,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837198 2024-11-14T14:00:37,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837198 2024-11-14T14:00:37,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837198) roll requested 2024-11-14T14:00:37,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837205 2024-11-14T14:00:37,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837205 2024-11-14T14:00:37,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837205) roll requested 2024-11-14T14:00:37,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837212 2024-11-14T14:00:37,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837212 2024-11-14T14:00:37,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837212) roll requested 2024-11-14T14:00:37,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837218 2024-11-14T14:00:37,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837218 2024-11-14T14:00:37,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837218) roll requested 2024-11-14T14:00:37,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837226 2024-11-14T14:00:37,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837226 2024-11-14T14:00:37,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837226) roll requested 2024-11-14T14:00:37,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837233 2024-11-14T14:00:37,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837233 2024-11-14T14:00:37,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837233) roll requested 2024-11-14T14:00:37,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837240 2024-11-14T14:00:37,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837240 2024-11-14T14:00:37,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837240) roll requested 2024-11-14T14:00:37,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837246 2024-11-14T14:00:37,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837246 2024-11-14T14:00:37,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837246) roll requested 2024-11-14T14:00:37,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837253 2024-11-14T14:00:37,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837253 2024-11-14T14:00:37,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837253) roll requested 2024-11-14T14:00:37,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837259 2024-11-14T14:00:37,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837259 2024-11-14T14:00:37,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837259) roll requested 2024-11-14T14:00:37,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837273 2024-11-14T14:00:37,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837273 2024-11-14T14:00:37,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837273) roll requested 2024-11-14T14:00:37,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837280 2024-11-14T14:00:37,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837280 2024-11-14T14:00:37,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837280) roll requested 2024-11-14T14:00:37,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837294 2024-11-14T14:00:37,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837294 2024-11-14T14:00:37,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837294) roll requested 2024-11-14T14:00:37,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837306 2024-11-14T14:00:37,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837306 2024-11-14T14:00:37,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837306) roll requested 2024-11-14T14:00:37,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837312 2024-11-14T14:00:37,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837312 2024-11-14T14:00:37,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837312) roll requested 2024-11-14T14:00:37,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837319 2024-11-14T14:00:37,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837319 2024-11-14T14:00:37,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837319) roll requested 2024-11-14T14:00:37,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837325 2024-11-14T14:00:37,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837325 2024-11-14T14:00:37,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837325) roll requested 2024-11-14T14:00:37,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837331 2024-11-14T14:00:37,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837331 2024-11-14T14:00:37,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837331) roll requested 2024-11-14T14:00:37,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837337 2024-11-14T14:00:37,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837337 2024-11-14T14:00:37,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837337) roll requested 2024-11-14T14:00:37,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837344 2024-11-14T14:00:37,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837344 2024-11-14T14:00:37,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837344) roll requested 2024-11-14T14:00:37,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837351 2024-11-14T14:00:37,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837351 2024-11-14T14:00:37,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837351) roll requested 2024-11-14T14:00:37,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837357 2024-11-14T14:00:37,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837357 2024-11-14T14:00:37,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837357) roll requested 2024-11-14T14:00:37,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837364 2024-11-14T14:00:37,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837364 2024-11-14T14:00:37,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837364) roll requested 2024-11-14T14:00:37,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837370 2024-11-14T14:00:37,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837370 2024-11-14T14:00:37,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837370) roll requested 2024-11-14T14:00:37,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837376 2024-11-14T14:00:37,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837376 2024-11-14T14:00:37,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837376) roll requested 2024-11-14T14:00:37,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837383 2024-11-14T14:00:37,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837383 2024-11-14T14:00:37,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837383) roll requested 2024-11-14T14:00:37,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837389 2024-11-14T14:00:37,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837389 2024-11-14T14:00:37,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837389) roll requested 2024-11-14T14:00:37,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837395 2024-11-14T14:00:37,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837395 2024-11-14T14:00:37,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837395) roll requested 2024-11-14T14:00:37,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837405 2024-11-14T14:00:37,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837405 2024-11-14T14:00:37,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837405) roll requested 2024-11-14T14:00:37,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837413 2024-11-14T14:00:37,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837413 2024-11-14T14:00:37,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837413) roll requested 2024-11-14T14:00:37,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837419 2024-11-14T14:00:37,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837419 2024-11-14T14:00:37,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837419) roll requested 2024-11-14T14:00:37,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837425 2024-11-14T14:00:37,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837425 2024-11-14T14:00:37,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837425) roll requested 2024-11-14T14:00:37,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837432 2024-11-14T14:00:37,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837432 2024-11-14T14:00:37,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837432) roll requested 2024-11-14T14:00:37,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837438 2024-11-14T14:00:37,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837438 2024-11-14T14:00:37,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837438) roll requested 2024-11-14T14:00:37,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837444 2024-11-14T14:00:37,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837444 2024-11-14T14:00:37,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837444) roll requested 2024-11-14T14:00:37,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837451 2024-11-14T14:00:37,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837451 2024-11-14T14:00:37,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837451) roll requested 2024-11-14T14:00:37,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837457 2024-11-14T14:00:37,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837457 2024-11-14T14:00:37,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837457) roll requested 2024-11-14T14:00:37,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837463 2024-11-14T14:00:37,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837463 2024-11-14T14:00:37,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837463) roll requested 2024-11-14T14:00:37,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837469 2024-11-14T14:00:37,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837469 2024-11-14T14:00:37,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837469) roll requested 2024-11-14T14:00:37,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837476 2024-11-14T14:00:37,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837476 2024-11-14T14:00:37,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837476) roll requested 2024-11-14T14:00:37,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837482 2024-11-14T14:00:37,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837482 2024-11-14T14:00:37,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837482) roll requested 2024-11-14T14:00:37,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837489 2024-11-14T14:00:37,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837489 2024-11-14T14:00:37,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837489) roll requested 2024-11-14T14:00:37,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837495 2024-11-14T14:00:37,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837495 2024-11-14T14:00:37,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837495) roll requested 2024-11-14T14:00:37,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837501 2024-11-14T14:00:37,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837501 2024-11-14T14:00:37,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837501) roll requested 2024-11-14T14:00:37,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837508 2024-11-14T14:00:37,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837508 2024-11-14T14:00:37,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837508) roll requested 2024-11-14T14:00:37,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837514 2024-11-14T14:00:37,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837514 2024-11-14T14:00:37,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837514) roll requested 2024-11-14T14:00:37,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837521 2024-11-14T14:00:37,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837521 2024-11-14T14:00:37,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837521) roll requested 2024-11-14T14:00:37,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837527 2024-11-14T14:00:37,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837527 2024-11-14T14:00:37,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837527) roll requested 2024-11-14T14:00:37,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837533 2024-11-14T14:00:37,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837533 2024-11-14T14:00:37,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837533) roll requested 2024-11-14T14:00:37,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837540 2024-11-14T14:00:37,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837540 2024-11-14T14:00:37,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837540) roll requested 2024-11-14T14:00:37,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837546 2024-11-14T14:00:37,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837546 2024-11-14T14:00:37,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837546) roll requested 2024-11-14T14:00:37,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837552 2024-11-14T14:00:37,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837552 2024-11-14T14:00:37,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837552) roll requested 2024-11-14T14:00:37,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837558 2024-11-14T14:00:37,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837558 2024-11-14T14:00:37,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837558) roll requested 2024-11-14T14:00:37,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837565 2024-11-14T14:00:37,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837565 2024-11-14T14:00:37,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837565) roll requested 2024-11-14T14:00:37,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837571 2024-11-14T14:00:37,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837571 2024-11-14T14:00:37,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837571) roll requested 2024-11-14T14:00:37,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837577 2024-11-14T14:00:37,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837577 2024-11-14T14:00:37,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837577) roll requested 2024-11-14T14:00:37,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837583 2024-11-14T14:00:37,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837583 2024-11-14T14:00:37,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837583) roll requested 2024-11-14T14:00:37,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837589 2024-11-14T14:00:37,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837589 2024-11-14T14:00:37,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837589) roll requested 2024-11-14T14:00:37,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837596 2024-11-14T14:00:37,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837596 2024-11-14T14:00:37,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837596) roll requested 2024-11-14T14:00:37,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837602 2024-11-14T14:00:37,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837602 2024-11-14T14:00:37,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837602) roll requested 2024-11-14T14:00:37,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837608 2024-11-14T14:00:37,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837608 2024-11-14T14:00:37,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837608) roll requested 2024-11-14T14:00:37,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837614 2024-11-14T14:00:37,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837614 2024-11-14T14:00:37,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837614) roll requested 2024-11-14T14:00:37,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837621 2024-11-14T14:00:37,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837621 2024-11-14T14:00:37,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837621) roll requested 2024-11-14T14:00:37,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837627 2024-11-14T14:00:37,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837627 2024-11-14T14:00:37,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837627) roll requested 2024-11-14T14:00:37,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837633 2024-11-14T14:00:37,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837633 2024-11-14T14:00:37,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837633) roll requested 2024-11-14T14:00:37,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837639 2024-11-14T14:00:37,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837639 2024-11-14T14:00:37,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837639) roll requested 2024-11-14T14:00:37,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837645 2024-11-14T14:00:37,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837645 2024-11-14T14:00:37,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837645) roll requested 2024-11-14T14:00:37,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837651 2024-11-14T14:00:37,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837651 2024-11-14T14:00:37,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837651) roll requested 2024-11-14T14:00:37,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837658 2024-11-14T14:00:37,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837658 2024-11-14T14:00:37,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837658) roll requested 2024-11-14T14:00:37,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837664 2024-11-14T14:00:37,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837664 2024-11-14T14:00:37,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837664) roll requested 2024-11-14T14:00:37,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837671 2024-11-14T14:00:37,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837671 2024-11-14T14:00:37,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837671) roll requested 2024-11-14T14:00:37,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837677 2024-11-14T14:00:37,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837677 2024-11-14T14:00:37,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837677) roll requested 2024-11-14T14:00:37,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837683 2024-11-14T14:00:37,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837683 2024-11-14T14:00:37,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837683) roll requested 2024-11-14T14:00:37,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837690 2024-11-14T14:00:37,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837690 2024-11-14T14:00:37,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837690) roll requested 2024-11-14T14:00:37,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837698 2024-11-14T14:00:37,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837698 2024-11-14T14:00:37,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837698) roll requested 2024-11-14T14:00:37,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837704 2024-11-14T14:00:37,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837704 2024-11-14T14:00:37,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837704) roll requested 2024-11-14T14:00:37,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837711 2024-11-14T14:00:37,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837711 2024-11-14T14:00:37,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837711) roll requested 2024-11-14T14:00:37,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837717 2024-11-14T14:00:37,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837717 2024-11-14T14:00:37,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837717) roll requested 2024-11-14T14:00:37,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837723 2024-11-14T14:00:37,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837723 2024-11-14T14:00:37,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837723) roll requested 2024-11-14T14:00:37,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837730 2024-11-14T14:00:37,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837730 2024-11-14T14:00:37,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837730) roll requested 2024-11-14T14:00:37,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837736 2024-11-14T14:00:37,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837736 2024-11-14T14:00:37,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837736) roll requested 2024-11-14T14:00:37,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837742 2024-11-14T14:00:37,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837742 2024-11-14T14:00:37,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837742) roll requested 2024-11-14T14:00:37,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837748 2024-11-14T14:00:37,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837748 2024-11-14T14:00:37,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837748) roll requested 2024-11-14T14:00:37,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837754 2024-11-14T14:00:37,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837754 2024-11-14T14:00:37,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837754) roll requested 2024-11-14T14:00:37,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837761 2024-11-14T14:00:37,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837761 2024-11-14T14:00:37,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837761) roll requested 2024-11-14T14:00:37,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837767 2024-11-14T14:00:37,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837767 2024-11-14T14:00:37,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837767) roll requested 2024-11-14T14:00:37,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837774 2024-11-14T14:00:37,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837774 2024-11-14T14:00:37,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837774) roll requested 2024-11-14T14:00:37,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837780 2024-11-14T14:00:37,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837780 2024-11-14T14:00:37,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837780) roll requested 2024-11-14T14:00:37,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837787 2024-11-14T14:00:37,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837787 2024-11-14T14:00:37,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837787) roll requested 2024-11-14T14:00:37,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837793 2024-11-14T14:00:37,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837793 2024-11-14T14:00:37,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837793) roll requested 2024-11-14T14:00:37,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837800 2024-11-14T14:00:37,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837800 2024-11-14T14:00:37,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837800) roll requested 2024-11-14T14:00:37,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837806 2024-11-14T14:00:37,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837806 2024-11-14T14:00:37,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837806) roll requested 2024-11-14T14:00:37,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837812 2024-11-14T14:00:37,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837812 2024-11-14T14:00:37,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837812) roll requested 2024-11-14T14:00:37,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837819 2024-11-14T14:00:37,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837819 2024-11-14T14:00:37,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837819) roll requested 2024-11-14T14:00:37,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837825 2024-11-14T14:00:37,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837825 2024-11-14T14:00:37,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837825) roll requested 2024-11-14T14:00:37,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837831 2024-11-14T14:00:37,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837831 2024-11-14T14:00:37,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837831) roll requested 2024-11-14T14:00:37,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837837 2024-11-14T14:00:37,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837837 2024-11-14T14:00:37,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837837) roll requested 2024-11-14T14:00:37,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837844 2024-11-14T14:00:37,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837844 2024-11-14T14:00:37,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837844) roll requested 2024-11-14T14:00:37,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837850 2024-11-14T14:00:37,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837850 2024-11-14T14:00:37,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837850) roll requested 2024-11-14T14:00:37,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837856 2024-11-14T14:00:37,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837856 2024-11-14T14:00:37,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837856) roll requested 2024-11-14T14:00:37,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837862 2024-11-14T14:00:37,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837862 2024-11-14T14:00:37,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837862) roll requested 2024-11-14T14:00:37,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837871 2024-11-14T14:00:37,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837871 2024-11-14T14:00:37,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837871) roll requested 2024-11-14T14:00:37,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837877 2024-11-14T14:00:37,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837877 2024-11-14T14:00:37,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837877) roll requested 2024-11-14T14:00:37,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837884 2024-11-14T14:00:37,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837884 2024-11-14T14:00:37,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837884) roll requested 2024-11-14T14:00:37,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837890 2024-11-14T14:00:37,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837890 2024-11-14T14:00:37,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837890) roll requested 2024-11-14T14:00:37,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837901 2024-11-14T14:00:37,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837901 2024-11-14T14:00:37,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837901) roll requested 2024-11-14T14:00:37,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837908 2024-11-14T14:00:37,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837908 2024-11-14T14:00:37,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837908) roll requested 2024-11-14T14:00:37,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837914 2024-11-14T14:00:37,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837914 2024-11-14T14:00:37,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837914) roll requested 2024-11-14T14:00:37,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837920 2024-11-14T14:00:37,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837920 2024-11-14T14:00:37,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837920) roll requested 2024-11-14T14:00:37,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837926 2024-11-14T14:00:37,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837926 2024-11-14T14:00:37,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837926) roll requested 2024-11-14T14:00:37,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837934 2024-11-14T14:00:37,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837934 2024-11-14T14:00:37,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837934) roll requested 2024-11-14T14:00:37,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837941 2024-11-14T14:00:37,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837941 2024-11-14T14:00:37,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837941) roll requested 2024-11-14T14:00:37,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837948 2024-11-14T14:00:37,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837948 2024-11-14T14:00:37,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837948) roll requested 2024-11-14T14:00:37,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837954 2024-11-14T14:00:37,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837954 2024-11-14T14:00:37,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837954) roll requested 2024-11-14T14:00:37,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837960 2024-11-14T14:00:37,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837960 2024-11-14T14:00:37,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837960) roll requested 2024-11-14T14:00:37,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837967 2024-11-14T14:00:37,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837967 2024-11-14T14:00:37,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837967) roll requested 2024-11-14T14:00:37,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837973 2024-11-14T14:00:37,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837973 2024-11-14T14:00:37,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837973) roll requested 2024-11-14T14:00:37,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837979 2024-11-14T14:00:37,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837979 2024-11-14T14:00:37,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837979) roll requested 2024-11-14T14:00:37,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837985 2024-11-14T14:00:37,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837985 2024-11-14T14:00:37,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837985) roll requested 2024-11-14T14:00:37,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837992 2024-11-14T14:00:37,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:37,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837992 2024-11-14T14:00:37,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:37,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:37,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837992) roll requested 2024-11-14T14:00:37,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592837998 2024-11-14T14:00:38,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837998 2024-11-14T14:00:38,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592837998) roll requested 2024-11-14T14:00:38,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838005 2024-11-14T14:00:38,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592837998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838005 2024-11-14T14:00:38,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838005) roll requested 2024-11-14T14:00:38,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838011 2024-11-14T14:00:38,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838011 2024-11-14T14:00:38,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838011) roll requested 2024-11-14T14:00:38,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838017 2024-11-14T14:00:38,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838017 2024-11-14T14:00:38,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838017) roll requested 2024-11-14T14:00:38,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838023 2024-11-14T14:00:38,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838023 2024-11-14T14:00:38,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838023) roll requested 2024-11-14T14:00:38,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838029 2024-11-14T14:00:38,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838029 2024-11-14T14:00:38,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838029) roll requested 2024-11-14T14:00:38,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838035 2024-11-14T14:00:38,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838035 2024-11-14T14:00:38,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838035) roll requested 2024-11-14T14:00:38,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838042 2024-11-14T14:00:38,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838042 2024-11-14T14:00:38,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838042) roll requested 2024-11-14T14:00:38,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838050 2024-11-14T14:00:38,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838050 2024-11-14T14:00:38,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838050) roll requested 2024-11-14T14:00:38,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838057 2024-11-14T14:00:38,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838057 2024-11-14T14:00:38,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838057) roll requested 2024-11-14T14:00:38,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838063 2024-11-14T14:00:38,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838063 2024-11-14T14:00:38,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838063) roll requested 2024-11-14T14:00:38,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838069 2024-11-14T14:00:38,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838069 2024-11-14T14:00:38,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838069) roll requested 2024-11-14T14:00:38,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838076 2024-11-14T14:00:38,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838076 2024-11-14T14:00:38,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838076) roll requested 2024-11-14T14:00:38,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838082 2024-11-14T14:00:38,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838082 2024-11-14T14:00:38,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838082) roll requested 2024-11-14T14:00:38,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838089 2024-11-14T14:00:38,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838089 2024-11-14T14:00:38,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838089) roll requested 2024-11-14T14:00:38,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838095 2024-11-14T14:00:38,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838095 2024-11-14T14:00:38,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838095) roll requested 2024-11-14T14:00:38,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838107 2024-11-14T14:00:38,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838107 2024-11-14T14:00:38,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838107) roll requested 2024-11-14T14:00:38,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838114 2024-11-14T14:00:38,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838114 2024-11-14T14:00:38,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838114) roll requested 2024-11-14T14:00:38,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838120 2024-11-14T14:00:38,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838120 2024-11-14T14:00:38,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838120) roll requested 2024-11-14T14:00:38,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838126 2024-11-14T14:00:38,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838126 2024-11-14T14:00:38,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838126) roll requested 2024-11-14T14:00:38,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838132 2024-11-14T14:00:38,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838132 2024-11-14T14:00:38,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838132) roll requested 2024-11-14T14:00:38,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838139 2024-11-14T14:00:38,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838139 2024-11-14T14:00:38,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838139) roll requested 2024-11-14T14:00:38,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838154 2024-11-14T14:00:38,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838154 2024-11-14T14:00:38,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838154) roll requested 2024-11-14T14:00:38,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838162 2024-11-14T14:00:38,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838162 2024-11-14T14:00:38,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838162) roll requested 2024-11-14T14:00:38,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838168 2024-11-14T14:00:38,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838168 2024-11-14T14:00:38,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838168) roll requested 2024-11-14T14:00:38,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838176 2024-11-14T14:00:38,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838176 2024-11-14T14:00:38,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838176) roll requested 2024-11-14T14:00:38,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838185 2024-11-14T14:00:38,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838185 2024-11-14T14:00:38,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838185) roll requested 2024-11-14T14:00:38,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838193 2024-11-14T14:00:38,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838193 2024-11-14T14:00:38,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838193) roll requested 2024-11-14T14:00:38,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838199 2024-11-14T14:00:38,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838199 2024-11-14T14:00:38,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838199) roll requested 2024-11-14T14:00:38,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838206 2024-11-14T14:00:38,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838206 2024-11-14T14:00:38,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838206) roll requested 2024-11-14T14:00:38,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838212 2024-11-14T14:00:38,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838212 2024-11-14T14:00:38,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838212) roll requested 2024-11-14T14:00:38,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838219 2024-11-14T14:00:38,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838219 2024-11-14T14:00:38,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838219) roll requested 2024-11-14T14:00:38,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838225 2024-11-14T14:00:38,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838225 2024-11-14T14:00:38,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838225) roll requested 2024-11-14T14:00:38,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838231 2024-11-14T14:00:38,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838231 2024-11-14T14:00:38,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838231) roll requested 2024-11-14T14:00:38,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838238 2024-11-14T14:00:38,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838238 2024-11-14T14:00:38,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838238) roll requested 2024-11-14T14:00:38,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838245 2024-11-14T14:00:38,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838245 2024-11-14T14:00:38,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838245) roll requested 2024-11-14T14:00:38,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838251 2024-11-14T14:00:38,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838251 2024-11-14T14:00:38,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838251) roll requested 2024-11-14T14:00:38,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838257 2024-11-14T14:00:38,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838257 2024-11-14T14:00:38,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838257) roll requested 2024-11-14T14:00:38,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838265 2024-11-14T14:00:38,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838265 2024-11-14T14:00:38,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838265) roll requested 2024-11-14T14:00:38,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838271 2024-11-14T14:00:38,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838271 2024-11-14T14:00:38,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838271) roll requested 2024-11-14T14:00:38,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838277 2024-11-14T14:00:38,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838277 2024-11-14T14:00:38,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838277) roll requested 2024-11-14T14:00:38,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838284 2024-11-14T14:00:38,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838284 2024-11-14T14:00:38,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838284) roll requested 2024-11-14T14:00:38,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838293 2024-11-14T14:00:38,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838293 2024-11-14T14:00:38,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838293) roll requested 2024-11-14T14:00:38,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838301 2024-11-14T14:00:38,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838301 2024-11-14T14:00:38,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838301) roll requested 2024-11-14T14:00:38,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838307 2024-11-14T14:00:38,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838307 2024-11-14T14:00:38,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838307) roll requested 2024-11-14T14:00:38,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838313 2024-11-14T14:00:38,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838313 2024-11-14T14:00:38,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838313) roll requested 2024-11-14T14:00:38,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838319 2024-11-14T14:00:38,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838319 2024-11-14T14:00:38,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838319) roll requested 2024-11-14T14:00:38,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838326 2024-11-14T14:00:38,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838326 2024-11-14T14:00:38,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838326) roll requested 2024-11-14T14:00:38,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838332 2024-11-14T14:00:38,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838332 2024-11-14T14:00:38,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838332) roll requested 2024-11-14T14:00:38,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838339 2024-11-14T14:00:38,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838339 2024-11-14T14:00:38,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838339) roll requested 2024-11-14T14:00:38,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838345 2024-11-14T14:00:38,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838345 2024-11-14T14:00:38,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838345) roll requested 2024-11-14T14:00:38,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838351 2024-11-14T14:00:38,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838351 2024-11-14T14:00:38,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838351) roll requested 2024-11-14T14:00:38,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838358 2024-11-14T14:00:38,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838358 2024-11-14T14:00:38,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838358) roll requested 2024-11-14T14:00:38,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838364 2024-11-14T14:00:38,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838364 2024-11-14T14:00:38,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838364) roll requested 2024-11-14T14:00:38,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838370 2024-11-14T14:00:38,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838370 2024-11-14T14:00:38,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838370) roll requested 2024-11-14T14:00:38,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838377 2024-11-14T14:00:38,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838377 2024-11-14T14:00:38,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838377) roll requested 2024-11-14T14:00:38,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838383 2024-11-14T14:00:38,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838383 2024-11-14T14:00:38,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838383) roll requested 2024-11-14T14:00:38,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838390 2024-11-14T14:00:38,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838390 2024-11-14T14:00:38,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838390) roll requested 2024-11-14T14:00:38,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838396 2024-11-14T14:00:38,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838396 2024-11-14T14:00:38,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838396) roll requested 2024-11-14T14:00:38,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838403 2024-11-14T14:00:38,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838403 2024-11-14T14:00:38,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838403) roll requested 2024-11-14T14:00:38,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838409 2024-11-14T14:00:38,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838409 2024-11-14T14:00:38,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838409) roll requested 2024-11-14T14:00:38,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838415 2024-11-14T14:00:38,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838415 2024-11-14T14:00:38,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838415) roll requested 2024-11-14T14:00:38,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838422 2024-11-14T14:00:38,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838422 2024-11-14T14:00:38,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838422) roll requested 2024-11-14T14:00:38,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838428 2024-11-14T14:00:38,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838428 2024-11-14T14:00:38,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838428) roll requested 2024-11-14T14:00:38,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838434 2024-11-14T14:00:38,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838434 2024-11-14T14:00:38,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838434) roll requested 2024-11-14T14:00:38,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838440 2024-11-14T14:00:38,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838440 2024-11-14T14:00:38,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838440) roll requested 2024-11-14T14:00:38,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838447 2024-11-14T14:00:38,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838447 2024-11-14T14:00:38,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838447) roll requested 2024-11-14T14:00:38,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838453 2024-11-14T14:00:38,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838453 2024-11-14T14:00:38,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838453) roll requested 2024-11-14T14:00:38,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838459 2024-11-14T14:00:38,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838459 2024-11-14T14:00:38,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838459) roll requested 2024-11-14T14:00:38,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838466 2024-11-14T14:00:38,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838466 2024-11-14T14:00:38,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838466) roll requested 2024-11-14T14:00:38,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838472 2024-11-14T14:00:38,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838472 2024-11-14T14:00:38,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838472) roll requested 2024-11-14T14:00:38,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838478 2024-11-14T14:00:38,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838478 2024-11-14T14:00:38,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838478) roll requested 2024-11-14T14:00:38,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838484 2024-11-14T14:00:38,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838484 2024-11-14T14:00:38,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838484) roll requested 2024-11-14T14:00:38,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838491 2024-11-14T14:00:38,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838491 2024-11-14T14:00:38,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838491) roll requested 2024-11-14T14:00:38,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838497 2024-11-14T14:00:38,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838497 2024-11-14T14:00:38,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838497) roll requested 2024-11-14T14:00:38,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838508 2024-11-14T14:00:38,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838508 2024-11-14T14:00:38,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838508) roll requested 2024-11-14T14:00:38,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838515 2024-11-14T14:00:38,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838515 2024-11-14T14:00:38,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838515) roll requested 2024-11-14T14:00:38,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838521 2024-11-14T14:00:38,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838521 2024-11-14T14:00:38,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838521) roll requested 2024-11-14T14:00:38,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838527 2024-11-14T14:00:38,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838527 2024-11-14T14:00:38,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838527) roll requested 2024-11-14T14:00:38,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838533 2024-11-14T14:00:38,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838533 2024-11-14T14:00:38,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838533) roll requested 2024-11-14T14:00:38,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838540 2024-11-14T14:00:38,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838540 2024-11-14T14:00:38,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838540) roll requested 2024-11-14T14:00:38,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838546 2024-11-14T14:00:38,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838546 2024-11-14T14:00:38,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838546) roll requested 2024-11-14T14:00:38,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838552 2024-11-14T14:00:38,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838552 2024-11-14T14:00:38,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838552) roll requested 2024-11-14T14:00:38,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838690 2024-11-14T14:00:38,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838690 2024-11-14T14:00:38,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838690) roll requested 2024-11-14T14:00:38,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838697 2024-11-14T14:00:38,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838697 2024-11-14T14:00:38,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838697) roll requested 2024-11-14T14:00:38,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838703 2024-11-14T14:00:38,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838703 2024-11-14T14:00:38,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838703) roll requested 2024-11-14T14:00:38,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838711 2024-11-14T14:00:38,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838711 2024-11-14T14:00:38,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838711) roll requested 2024-11-14T14:00:38,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838718 2024-11-14T14:00:38,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838718 2024-11-14T14:00:38,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838718) roll requested 2024-11-14T14:00:38,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838724 2024-11-14T14:00:38,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838724 2024-11-14T14:00:38,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838724) roll requested 2024-11-14T14:00:38,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838730 2024-11-14T14:00:38,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838730 2024-11-14T14:00:38,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838730) roll requested 2024-11-14T14:00:38,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838737 2024-11-14T14:00:38,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838737 2024-11-14T14:00:38,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838737) roll requested 2024-11-14T14:00:38,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838743 2024-11-14T14:00:38,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838743 2024-11-14T14:00:38,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838743) roll requested 2024-11-14T14:00:38,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838750 2024-11-14T14:00:38,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838750 2024-11-14T14:00:38,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838750) roll requested 2024-11-14T14:00:38,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838756 2024-11-14T14:00:38,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838756 2024-11-14T14:00:38,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838756) roll requested 2024-11-14T14:00:38,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838762 2024-11-14T14:00:38,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838762 2024-11-14T14:00:38,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838762) roll requested 2024-11-14T14:00:38,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838768 2024-11-14T14:00:38,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838768 2024-11-14T14:00:38,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838768) roll requested 2024-11-14T14:00:38,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838775 2024-11-14T14:00:38,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838775 2024-11-14T14:00:38,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838775) roll requested 2024-11-14T14:00:38,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838781 2024-11-14T14:00:38,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838781 2024-11-14T14:00:38,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838781) roll requested 2024-11-14T14:00:38,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838787 2024-11-14T14:00:38,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838787 2024-11-14T14:00:38,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838787) roll requested 2024-11-14T14:00:38,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838794 2024-11-14T14:00:38,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838794 2024-11-14T14:00:38,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838794) roll requested 2024-11-14T14:00:38,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838800 2024-11-14T14:00:38,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838800 2024-11-14T14:00:38,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838800) roll requested 2024-11-14T14:00:38,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838806 2024-11-14T14:00:38,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838806 2024-11-14T14:00:38,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838806) roll requested 2024-11-14T14:00:38,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838813 2024-11-14T14:00:38,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838813 2024-11-14T14:00:38,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838813) roll requested 2024-11-14T14:00:38,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838819 2024-11-14T14:00:38,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838819 2024-11-14T14:00:38,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838819) roll requested 2024-11-14T14:00:38,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838826 2024-11-14T14:00:38,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838826 2024-11-14T14:00:38,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838826) roll requested 2024-11-14T14:00:38,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838832 2024-11-14T14:00:38,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838832 2024-11-14T14:00:38,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838832) roll requested 2024-11-14T14:00:38,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838838 2024-11-14T14:00:38,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838838 2024-11-14T14:00:38,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838838) roll requested 2024-11-14T14:00:38,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838844 2024-11-14T14:00:38,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838844 2024-11-14T14:00:38,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838844) roll requested 2024-11-14T14:00:38,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838850 2024-11-14T14:00:38,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838850 2024-11-14T14:00:38,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838850) roll requested 2024-11-14T14:00:38,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838857 2024-11-14T14:00:38,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838857 2024-11-14T14:00:38,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838857) roll requested 2024-11-14T14:00:38,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838863 2024-11-14T14:00:38,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838863 2024-11-14T14:00:38,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838863) roll requested 2024-11-14T14:00:38,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838869 2024-11-14T14:00:38,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838869 2024-11-14T14:00:38,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838869) roll requested 2024-11-14T14:00:38,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838876 2024-11-14T14:00:38,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838876 2024-11-14T14:00:38,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838876) roll requested 2024-11-14T14:00:38,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838882 2024-11-14T14:00:38,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838882 2024-11-14T14:00:38,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838882) roll requested 2024-11-14T14:00:38,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838889 2024-11-14T14:00:38,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838889 2024-11-14T14:00:38,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838889) roll requested 2024-11-14T14:00:38,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838895 2024-11-14T14:00:38,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838895 2024-11-14T14:00:38,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838895) roll requested 2024-11-14T14:00:38,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838901 2024-11-14T14:00:38,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838901 2024-11-14T14:00:38,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838901) roll requested 2024-11-14T14:00:38,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838908 2024-11-14T14:00:38,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838908 2024-11-14T14:00:38,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838908) roll requested 2024-11-14T14:00:38,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838914 2024-11-14T14:00:38,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838914 2024-11-14T14:00:38,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838914) roll requested 2024-11-14T14:00:38,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838920 2024-11-14T14:00:38,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838920 2024-11-14T14:00:38,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838920) roll requested 2024-11-14T14:00:38,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838926 2024-11-14T14:00:38,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838926 2024-11-14T14:00:38,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838926) roll requested 2024-11-14T14:00:38,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838933 2024-11-14T14:00:38,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838933 2024-11-14T14:00:38,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838933) roll requested 2024-11-14T14:00:38,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838939 2024-11-14T14:00:38,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838939 2024-11-14T14:00:38,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838939) roll requested 2024-11-14T14:00:38,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838945 2024-11-14T14:00:38,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838945 2024-11-14T14:00:38,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838945) roll requested 2024-11-14T14:00:38,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838951 2024-11-14T14:00:38,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838951 2024-11-14T14:00:38,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838951) roll requested 2024-11-14T14:00:38,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838958 2024-11-14T14:00:38,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838958 2024-11-14T14:00:38,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838958) roll requested 2024-11-14T14:00:38,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838964 2024-11-14T14:00:38,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838964 2024-11-14T14:00:38,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838964) roll requested 2024-11-14T14:00:38,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838971 2024-11-14T14:00:38,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838971 2024-11-14T14:00:38,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838971) roll requested 2024-11-14T14:00:38,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838977 2024-11-14T14:00:38,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838977 2024-11-14T14:00:38,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838977) roll requested 2024-11-14T14:00:38,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838983 2024-11-14T14:00:38,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838983 2024-11-14T14:00:38,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838983) roll requested 2024-11-14T14:00:38,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838989 2024-11-14T14:00:38,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:38,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838989 2024-11-14T14:00:38,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:38,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:38,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838989) roll requested 2024-11-14T14:00:38,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592838995 2024-11-14T14:00:39,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838995 2024-11-14T14:00:39,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592838995) roll requested 2024-11-14T14:00:39,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839001 2024-11-14T14:00:39,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592838995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839001 2024-11-14T14:00:39,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839001) roll requested 2024-11-14T14:00:39,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839008 2024-11-14T14:00:39,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839008 2024-11-14T14:00:39,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839008) roll requested 2024-11-14T14:00:39,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839014 2024-11-14T14:00:39,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839014 2024-11-14T14:00:39,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839014) roll requested 2024-11-14T14:00:39,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839020 2024-11-14T14:00:39,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839020 2024-11-14T14:00:39,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839020) roll requested 2024-11-14T14:00:39,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839026 2024-11-14T14:00:39,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839026 2024-11-14T14:00:39,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839026) roll requested 2024-11-14T14:00:39,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839033 2024-11-14T14:00:39,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839033 2024-11-14T14:00:39,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839033) roll requested 2024-11-14T14:00:39,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839039 2024-11-14T14:00:39,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839039 2024-11-14T14:00:39,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839039) roll requested 2024-11-14T14:00:39,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839047 2024-11-14T14:00:39,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839047 2024-11-14T14:00:39,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839047) roll requested 2024-11-14T14:00:39,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839054 2024-11-14T14:00:39,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839054 2024-11-14T14:00:39,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839054) roll requested 2024-11-14T14:00:39,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839061 2024-11-14T14:00:39,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839061 2024-11-14T14:00:39,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839061) roll requested 2024-11-14T14:00:39,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839067 2024-11-14T14:00:39,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839067 2024-11-14T14:00:39,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839067) roll requested 2024-11-14T14:00:39,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839073 2024-11-14T14:00:39,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839073 2024-11-14T14:00:39,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839073) roll requested 2024-11-14T14:00:39,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839080 2024-11-14T14:00:39,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839080 2024-11-14T14:00:39,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839080) roll requested 2024-11-14T14:00:39,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839086 2024-11-14T14:00:39,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839086 2024-11-14T14:00:39,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839086) roll requested 2024-11-14T14:00:39,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839092 2024-11-14T14:00:39,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839092 2024-11-14T14:00:39,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839092) roll requested 2024-11-14T14:00:39,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839098 2024-11-14T14:00:39,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839098 2024-11-14T14:00:39,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839098) roll requested 2024-11-14T14:00:39,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839113 2024-11-14T14:00:39,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839113 2024-11-14T14:00:39,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839113) roll requested 2024-11-14T14:00:39,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839120 2024-11-14T14:00:39,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839120 2024-11-14T14:00:39,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839120) roll requested 2024-11-14T14:00:39,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839127 2024-11-14T14:00:39,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839127 2024-11-14T14:00:39,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839127) roll requested 2024-11-14T14:00:39,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839135 2024-11-14T14:00:39,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839135 2024-11-14T14:00:39,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839135) roll requested 2024-11-14T14:00:39,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839141 2024-11-14T14:00:39,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839141 2024-11-14T14:00:39,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839141) roll requested 2024-11-14T14:00:39,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839150 2024-11-14T14:00:39,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839150 2024-11-14T14:00:39,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839150) roll requested 2024-11-14T14:00:39,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839156 2024-11-14T14:00:39,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839156 2024-11-14T14:00:39,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839156) roll requested 2024-11-14T14:00:39,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839162 2024-11-14T14:00:39,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839162 2024-11-14T14:00:39,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839162) roll requested 2024-11-14T14:00:39,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839169 2024-11-14T14:00:39,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839169 2024-11-14T14:00:39,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839169) roll requested 2024-11-14T14:00:39,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839175 2024-11-14T14:00:39,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839175 2024-11-14T14:00:39,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839175) roll requested 2024-11-14T14:00:39,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839181 2024-11-14T14:00:39,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839181 2024-11-14T14:00:39,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839181) roll requested 2024-11-14T14:00:39,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839188 2024-11-14T14:00:39,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839188 2024-11-14T14:00:39,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839188) roll requested 2024-11-14T14:00:39,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839195 2024-11-14T14:00:39,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839195 2024-11-14T14:00:39,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839195) roll requested 2024-11-14T14:00:39,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839202 2024-11-14T14:00:39,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839202 2024-11-14T14:00:39,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839202) roll requested 2024-11-14T14:00:39,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839208 2024-11-14T14:00:39,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839208 2024-11-14T14:00:39,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839208) roll requested 2024-11-14T14:00:39,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839214 2024-11-14T14:00:39,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839214 2024-11-14T14:00:39,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839214) roll requested 2024-11-14T14:00:39,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839220 2024-11-14T14:00:39,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839220 2024-11-14T14:00:39,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839220) roll requested 2024-11-14T14:00:39,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839230 2024-11-14T14:00:39,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839230 2024-11-14T14:00:39,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839230) roll requested 2024-11-14T14:00:39,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839237 2024-11-14T14:00:39,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839237 2024-11-14T14:00:39,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839237) roll requested 2024-11-14T14:00:39,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839243 2024-11-14T14:00:39,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839243 2024-11-14T14:00:39,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839243) roll requested 2024-11-14T14:00:39,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839253 2024-11-14T14:00:39,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839253 2024-11-14T14:00:39,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839253) roll requested 2024-11-14T14:00:39,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839261 2024-11-14T14:00:39,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839261 2024-11-14T14:00:39,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839261) roll requested 2024-11-14T14:00:39,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839269 2024-11-14T14:00:39,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839269 2024-11-14T14:00:39,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839269) roll requested 2024-11-14T14:00:39,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839277 2024-11-14T14:00:39,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839277 2024-11-14T14:00:39,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839277) roll requested 2024-11-14T14:00:39,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839289 2024-11-14T14:00:39,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839289 2024-11-14T14:00:39,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839289) roll requested 2024-11-14T14:00:39,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839302 2024-11-14T14:00:39,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839302 2024-11-14T14:00:39,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839302) roll requested 2024-11-14T14:00:39,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839312 2024-11-14T14:00:39,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839312 2024-11-14T14:00:39,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839312) roll requested 2024-11-14T14:00:39,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839324 2024-11-14T14:00:39,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839324 2024-11-14T14:00:39,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839324) roll requested 2024-11-14T14:00:39,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839331 2024-11-14T14:00:39,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839331 2024-11-14T14:00:39,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839331) roll requested 2024-11-14T14:00:39,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839337 2024-11-14T14:00:39,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839337 2024-11-14T14:00:39,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839337) roll requested 2024-11-14T14:00:39,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839345 2024-11-14T14:00:39,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839345 2024-11-14T14:00:39,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839345) roll requested 2024-11-14T14:00:39,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839351 2024-11-14T14:00:39,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839351 2024-11-14T14:00:39,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839351) roll requested 2024-11-14T14:00:39,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839358 2024-11-14T14:00:39,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839358 2024-11-14T14:00:39,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839358) roll requested 2024-11-14T14:00:39,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839364 2024-11-14T14:00:39,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839364 2024-11-14T14:00:39,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839364) roll requested 2024-11-14T14:00:39,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839370 2024-11-14T14:00:39,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839370 2024-11-14T14:00:39,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839370) roll requested 2024-11-14T14:00:39,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839376 2024-11-14T14:00:39,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839376 2024-11-14T14:00:39,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839376) roll requested 2024-11-14T14:00:39,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839383 2024-11-14T14:00:39,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839383 2024-11-14T14:00:39,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839383) roll requested 2024-11-14T14:00:39,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839389 2024-11-14T14:00:39,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839389 2024-11-14T14:00:39,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839389) roll requested 2024-11-14T14:00:39,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839396 2024-11-14T14:00:39,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839396 2024-11-14T14:00:39,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839396) roll requested 2024-11-14T14:00:39,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839402 2024-11-14T14:00:39,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839402 2024-11-14T14:00:39,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839402) roll requested 2024-11-14T14:00:39,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839408 2024-11-14T14:00:39,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839408 2024-11-14T14:00:39,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839408) roll requested 2024-11-14T14:00:39,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839415 2024-11-14T14:00:39,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839415 2024-11-14T14:00:39,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839415) roll requested 2024-11-14T14:00:39,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839421 2024-11-14T14:00:39,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839421 2024-11-14T14:00:39,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839421) roll requested 2024-11-14T14:00:39,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839427 2024-11-14T14:00:39,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839427 2024-11-14T14:00:39,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839427) roll requested 2024-11-14T14:00:39,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839433 2024-11-14T14:00:39,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839433 2024-11-14T14:00:39,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839433) roll requested 2024-11-14T14:00:39,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839440 2024-11-14T14:00:39,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839440 2024-11-14T14:00:39,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839440) roll requested 2024-11-14T14:00:39,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839447 2024-11-14T14:00:39,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839447 2024-11-14T14:00:39,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839447) roll requested 2024-11-14T14:00:39,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839454 2024-11-14T14:00:39,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839454 2024-11-14T14:00:39,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839454) roll requested 2024-11-14T14:00:39,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839462 2024-11-14T14:00:39,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839462 2024-11-14T14:00:39,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839462) roll requested 2024-11-14T14:00:39,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839469 2024-11-14T14:00:39,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839469 2024-11-14T14:00:39,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839469) roll requested 2024-11-14T14:00:39,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839475 2024-11-14T14:00:39,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839475 2024-11-14T14:00:39,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839475) roll requested 2024-11-14T14:00:39,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839482 2024-11-14T14:00:39,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839482 2024-11-14T14:00:39,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839482) roll requested 2024-11-14T14:00:39,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839488 2024-11-14T14:00:39,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839488 2024-11-14T14:00:39,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839488) roll requested 2024-11-14T14:00:39,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839497 2024-11-14T14:00:39,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839497 2024-11-14T14:00:39,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839497) roll requested 2024-11-14T14:00:39,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839505 2024-11-14T14:00:39,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839505 2024-11-14T14:00:39,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839505) roll requested 2024-11-14T14:00:39,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839511 2024-11-14T14:00:39,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839511 2024-11-14T14:00:39,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839511) roll requested 2024-11-14T14:00:39,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839518 2024-11-14T14:00:39,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839518 2024-11-14T14:00:39,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839518) roll requested 2024-11-14T14:00:39,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839525 2024-11-14T14:00:39,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839525 2024-11-14T14:00:39,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839525) roll requested 2024-11-14T14:00:39,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839531 2024-11-14T14:00:39,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839531 2024-11-14T14:00:39,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839531) roll requested 2024-11-14T14:00:39,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839537 2024-11-14T14:00:39,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839537 2024-11-14T14:00:39,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839537) roll requested 2024-11-14T14:00:39,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839546 2024-11-14T14:00:39,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839546 2024-11-14T14:00:39,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839546) roll requested 2024-11-14T14:00:39,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839553 2024-11-14T14:00:39,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839553 2024-11-14T14:00:39,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839553) roll requested 2024-11-14T14:00:39,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839559 2024-11-14T14:00:39,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839559 2024-11-14T14:00:39,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839559) roll requested 2024-11-14T14:00:39,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839565 2024-11-14T14:00:39,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839565 2024-11-14T14:00:39,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839565) roll requested 2024-11-14T14:00:39,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839572 2024-11-14T14:00:39,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839572 2024-11-14T14:00:39,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839572) roll requested 2024-11-14T14:00:39,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839579 2024-11-14T14:00:39,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839579 2024-11-14T14:00:39,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839579) roll requested 2024-11-14T14:00:39,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839585 2024-11-14T14:00:39,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839585 2024-11-14T14:00:39,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839585) roll requested 2024-11-14T14:00:39,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839591 2024-11-14T14:00:39,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839591 2024-11-14T14:00:39,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839585 is not closed yet, will try archiving it next time 2024-11-14T14:00:39,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839591) roll requested 2024-11-14T14:00:39,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839598 2024-11-14T14:00:39,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839598 2024-11-14T14:00:39,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839598) roll requested 2024-11-14T14:00:39,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839612 2024-11-14T14:00:39,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839612 2024-11-14T14:00:39,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839612) roll requested 2024-11-14T14:00:39,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839618 2024-11-14T14:00:39,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839618 2024-11-14T14:00:39,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839618) roll requested 2024-11-14T14:00:39,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839625 2024-11-14T14:00:39,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839625 2024-11-14T14:00:39,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839625) roll requested 2024-11-14T14:00:39,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839631 2024-11-14T14:00:39,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839631 2024-11-14T14:00:39,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839631) roll requested 2024-11-14T14:00:39,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839637 2024-11-14T14:00:39,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839637 2024-11-14T14:00:39,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839637) roll requested 2024-11-14T14:00:39,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839643 2024-11-14T14:00:39,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839643 2024-11-14T14:00:39,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839643) roll requested 2024-11-14T14:00:39,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839650 2024-11-14T14:00:39,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839650 2024-11-14T14:00:39,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839650) roll requested 2024-11-14T14:00:39,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839657 2024-11-14T14:00:39,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839657 2024-11-14T14:00:39,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839657) roll requested 2024-11-14T14:00:39,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839663 2024-11-14T14:00:39,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839663 2024-11-14T14:00:39,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839663) roll requested 2024-11-14T14:00:39,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839669 2024-11-14T14:00:39,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839669 2024-11-14T14:00:39,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839669) roll requested 2024-11-14T14:00:39,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839676 2024-11-14T14:00:39,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839676 2024-11-14T14:00:39,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839676) roll requested 2024-11-14T14:00:39,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839682 2024-11-14T14:00:39,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839682 2024-11-14T14:00:39,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839682) roll requested 2024-11-14T14:00:39,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839688 2024-11-14T14:00:39,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839688 2024-11-14T14:00:39,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839688) roll requested 2024-11-14T14:00:39,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839695 2024-11-14T14:00:39,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839695 2024-11-14T14:00:39,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839695) roll requested 2024-11-14T14:00:39,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839701 2024-11-14T14:00:39,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839701 2024-11-14T14:00:39,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839701) roll requested 2024-11-14T14:00:39,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839708 2024-11-14T14:00:39,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839708 2024-11-14T14:00:39,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839708) roll requested 2024-11-14T14:00:39,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839714 2024-11-14T14:00:39,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839714 2024-11-14T14:00:39,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839714) roll requested 2024-11-14T14:00:39,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839720 2024-11-14T14:00:39,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839720 2024-11-14T14:00:39,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839720) roll requested 2024-11-14T14:00:39,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839727 2024-11-14T14:00:39,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839727 2024-11-14T14:00:39,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839727) roll requested 2024-11-14T14:00:39,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839733 2024-11-14T14:00:39,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839733 2024-11-14T14:00:39,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839733) roll requested 2024-11-14T14:00:39,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839740 2024-11-14T14:00:39,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839740 2024-11-14T14:00:39,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839740) roll requested 2024-11-14T14:00:39,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839746 2024-11-14T14:00:39,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839746 2024-11-14T14:00:39,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839746) roll requested 2024-11-14T14:00:39,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839752 2024-11-14T14:00:39,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839752 2024-11-14T14:00:39,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839752) roll requested 2024-11-14T14:00:39,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839758 2024-11-14T14:00:39,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839758 2024-11-14T14:00:39,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839758) roll requested 2024-11-14T14:00:39,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839765 2024-11-14T14:00:39,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839765 2024-11-14T14:00:39,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839765) roll requested 2024-11-14T14:00:39,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839771 2024-11-14T14:00:39,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839771 2024-11-14T14:00:39,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839771) roll requested 2024-11-14T14:00:39,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839777 2024-11-14T14:00:39,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839777 2024-11-14T14:00:39,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839777) roll requested 2024-11-14T14:00:39,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839783 2024-11-14T14:00:39,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839783 2024-11-14T14:00:39,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839783) roll requested 2024-11-14T14:00:39,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839789 2024-11-14T14:00:39,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839789 2024-11-14T14:00:39,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839789) roll requested 2024-11-14T14:00:39,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839863 2024-11-14T14:00:39,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839863 2024-11-14T14:00:39,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839863) roll requested 2024-11-14T14:00:39,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839869 2024-11-14T14:00:39,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839869 2024-11-14T14:00:39,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839869) roll requested 2024-11-14T14:00:39,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839876 2024-11-14T14:00:39,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839876 2024-11-14T14:00:39,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839876) roll requested 2024-11-14T14:00:39,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839882 2024-11-14T14:00:39,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839882 2024-11-14T14:00:39,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839882) roll requested 2024-11-14T14:00:39,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839888 2024-11-14T14:00:39,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839888 2024-11-14T14:00:39,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839888) roll requested 2024-11-14T14:00:39,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839894 2024-11-14T14:00:39,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839894 2024-11-14T14:00:39,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839894) roll requested 2024-11-14T14:00:39,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839901 2024-11-14T14:00:39,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839901 2024-11-14T14:00:39,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839901) roll requested 2024-11-14T14:00:39,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839907 2024-11-14T14:00:39,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839907 2024-11-14T14:00:39,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839907) roll requested 2024-11-14T14:00:39,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839913 2024-11-14T14:00:39,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839913 2024-11-14T14:00:39,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839913) roll requested 2024-11-14T14:00:39,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839919 2024-11-14T14:00:39,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839919 2024-11-14T14:00:39,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839919) roll requested 2024-11-14T14:00:39,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839925 2024-11-14T14:00:39,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839925 2024-11-14T14:00:39,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839925) roll requested 2024-11-14T14:00:39,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839931 2024-11-14T14:00:39,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839931 2024-11-14T14:00:39,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839931) roll requested 2024-11-14T14:00:39,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839941 2024-11-14T14:00:39,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839941 2024-11-14T14:00:39,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839941) roll requested 2024-11-14T14:00:39,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839947 2024-11-14T14:00:39,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839947 2024-11-14T14:00:39,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839947) roll requested 2024-11-14T14:00:39,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839954 2024-11-14T14:00:39,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839954 2024-11-14T14:00:39,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839954) roll requested 2024-11-14T14:00:39,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839961 2024-11-14T14:00:39,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839961 2024-11-14T14:00:39,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839961) roll requested 2024-11-14T14:00:39,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839967 2024-11-14T14:00:39,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839967 2024-11-14T14:00:39,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839967) roll requested 2024-11-14T14:00:39,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839973 2024-11-14T14:00:39,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839973 2024-11-14T14:00:39,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839973) roll requested 2024-11-14T14:00:39,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839980 2024-11-14T14:00:39,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839980 2024-11-14T14:00:39,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839980) roll requested 2024-11-14T14:00:39,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839986 2024-11-14T14:00:39,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839986 2024-11-14T14:00:39,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839986) roll requested 2024-11-14T14:00:39,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839992 2024-11-14T14:00:39,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:39,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839992 2024-11-14T14:00:39,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:39,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:39,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839992) roll requested 2024-11-14T14:00:39,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592839998 2024-11-14T14:00:40,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839998 2024-11-14T14:00:40,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592839998) roll requested 2024-11-14T14:00:40,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840004 2024-11-14T14:00:40,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592839998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840004 2024-11-14T14:00:40,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840004) roll requested 2024-11-14T14:00:40,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840010 2024-11-14T14:00:40,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840010 2024-11-14T14:00:40,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840010) roll requested 2024-11-14T14:00:40,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840017 2024-11-14T14:00:40,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840017 2024-11-14T14:00:40,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840017) roll requested 2024-11-14T14:00:40,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840023 2024-11-14T14:00:40,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840023 2024-11-14T14:00:40,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840023) roll requested 2024-11-14T14:00:40,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840029 2024-11-14T14:00:40,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840029 2024-11-14T14:00:40,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840029) roll requested 2024-11-14T14:00:40,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840035 2024-11-14T14:00:40,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840035 2024-11-14T14:00:40,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840035) roll requested 2024-11-14T14:00:40,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840041 2024-11-14T14:00:40,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840041 2024-11-14T14:00:40,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840041) roll requested 2024-11-14T14:00:40,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840048 2024-11-14T14:00:40,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840048 2024-11-14T14:00:40,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840048) roll requested 2024-11-14T14:00:40,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840054 2024-11-14T14:00:40,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840054 2024-11-14T14:00:40,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840054) roll requested 2024-11-14T14:00:40,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840065 2024-11-14T14:00:40,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840065 2024-11-14T14:00:40,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840065) roll requested 2024-11-14T14:00:40,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840071 2024-11-14T14:00:40,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840071 2024-11-14T14:00:40,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840071) roll requested 2024-11-14T14:00:40,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840077 2024-11-14T14:00:40,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840077 2024-11-14T14:00:40,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840077) roll requested 2024-11-14T14:00:40,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840083 2024-11-14T14:00:40,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840083 2024-11-14T14:00:40,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840083) roll requested 2024-11-14T14:00:40,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840090 2024-11-14T14:00:40,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840090 2024-11-14T14:00:40,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840090) roll requested 2024-11-14T14:00:40,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840096 2024-11-14T14:00:40,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840096 2024-11-14T14:00:40,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840096) roll requested 2024-11-14T14:00:40,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840102 2024-11-14T14:00:40,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840102 2024-11-14T14:00:40,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840102) roll requested 2024-11-14T14:00:40,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840108 2024-11-14T14:00:40,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840108 2024-11-14T14:00:40,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840108) roll requested 2024-11-14T14:00:40,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840114 2024-11-14T14:00:40,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840114 2024-11-14T14:00:40,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840114) roll requested 2024-11-14T14:00:40,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840121 2024-11-14T14:00:40,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840121 2024-11-14T14:00:40,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840121) roll requested 2024-11-14T14:00:40,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840127 2024-11-14T14:00:40,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840127 2024-11-14T14:00:40,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840127) roll requested 2024-11-14T14:00:40,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840133 2024-11-14T14:00:40,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840133 2024-11-14T14:00:40,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840133) roll requested 2024-11-14T14:00:40,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840139 2024-11-14T14:00:40,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840139 2024-11-14T14:00:40,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840139) roll requested 2024-11-14T14:00:40,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840146 2024-11-14T14:00:40,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840146 2024-11-14T14:00:40,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840146) roll requested 2024-11-14T14:00:40,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840152 2024-11-14T14:00:40,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840152 2024-11-14T14:00:40,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840152) roll requested 2024-11-14T14:00:40,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840158 2024-11-14T14:00:40,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840158 2024-11-14T14:00:40,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840158) roll requested 2024-11-14T14:00:40,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840164 2024-11-14T14:00:40,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840164 2024-11-14T14:00:40,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840164) roll requested 2024-11-14T14:00:40,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840170 2024-11-14T14:00:40,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840170 2024-11-14T14:00:40,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840170) roll requested 2024-11-14T14:00:40,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840177 2024-11-14T14:00:40,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840177 2024-11-14T14:00:40,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840177) roll requested 2024-11-14T14:00:40,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840183 2024-11-14T14:00:40,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840183 2024-11-14T14:00:40,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840183) roll requested 2024-11-14T14:00:40,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840189 2024-11-14T14:00:40,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840189 2024-11-14T14:00:40,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840189) roll requested 2024-11-14T14:00:40,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840195 2024-11-14T14:00:40,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840195 2024-11-14T14:00:40,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840195) roll requested 2024-11-14T14:00:40,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840201 2024-11-14T14:00:40,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840201 2024-11-14T14:00:40,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840201) roll requested 2024-11-14T14:00:40,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840207 2024-11-14T14:00:40,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840207 2024-11-14T14:00:40,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840207) roll requested 2024-11-14T14:00:40,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840214 2024-11-14T14:00:40,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840214 2024-11-14T14:00:40,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840214) roll requested 2024-11-14T14:00:40,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840224 2024-11-14T14:00:40,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840224 2024-11-14T14:00:40,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840224) roll requested 2024-11-14T14:00:40,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840232 2024-11-14T14:00:40,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840232 2024-11-14T14:00:40,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840232) roll requested 2024-11-14T14:00:40,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840240 2024-11-14T14:00:40,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840240 2024-11-14T14:00:40,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840240) roll requested 2024-11-14T14:00:40,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840253 2024-11-14T14:00:40,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840253 2024-11-14T14:00:40,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840253) roll requested 2024-11-14T14:00:40,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840266 2024-11-14T14:00:40,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840266 2024-11-14T14:00:40,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840266) roll requested 2024-11-14T14:00:40,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840272 2024-11-14T14:00:40,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840272 2024-11-14T14:00:40,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840272) roll requested 2024-11-14T14:00:40,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840286 2024-11-14T14:00:40,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840286 2024-11-14T14:00:40,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840286) roll requested 2024-11-14T14:00:40,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840293 2024-11-14T14:00:40,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840293 2024-11-14T14:00:40,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840293) roll requested 2024-11-14T14:00:40,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840300 2024-11-14T14:00:40,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840300 2024-11-14T14:00:40,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840300) roll requested 2024-11-14T14:00:40,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840306 2024-11-14T14:00:40,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840306 2024-11-14T14:00:40,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840306) roll requested 2024-11-14T14:00:40,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840312 2024-11-14T14:00:40,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840312 2024-11-14T14:00:40,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840312) roll requested 2024-11-14T14:00:40,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840318 2024-11-14T14:00:40,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840318 2024-11-14T14:00:40,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840318) roll requested 2024-11-14T14:00:40,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840325 2024-11-14T14:00:40,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840325 2024-11-14T14:00:40,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840325) roll requested 2024-11-14T14:00:40,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840331 2024-11-14T14:00:40,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840331 2024-11-14T14:00:40,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840331) roll requested 2024-11-14T14:00:40,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840337 2024-11-14T14:00:40,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840337 2024-11-14T14:00:40,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840337) roll requested 2024-11-14T14:00:40,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840343 2024-11-14T14:00:40,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840343 2024-11-14T14:00:40,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840343) roll requested 2024-11-14T14:00:40,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840350 2024-11-14T14:00:40,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840350 2024-11-14T14:00:40,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840350) roll requested 2024-11-14T14:00:40,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840356 2024-11-14T14:00:40,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840356 2024-11-14T14:00:40,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840356) roll requested 2024-11-14T14:00:40,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840363 2024-11-14T14:00:40,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840363 2024-11-14T14:00:40,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840363) roll requested 2024-11-14T14:00:40,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840369 2024-11-14T14:00:40,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840369 2024-11-14T14:00:40,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840369) roll requested 2024-11-14T14:00:40,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840375 2024-11-14T14:00:40,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840375 2024-11-14T14:00:40,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840375) roll requested 2024-11-14T14:00:40,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840381 2024-11-14T14:00:40,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840381 2024-11-14T14:00:40,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840381) roll requested 2024-11-14T14:00:40,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840387 2024-11-14T14:00:40,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840387 2024-11-14T14:00:40,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840387) roll requested 2024-11-14T14:00:40,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840394 2024-11-14T14:00:40,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840394 2024-11-14T14:00:40,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840394) roll requested 2024-11-14T14:00:40,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840403 2024-11-14T14:00:40,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840403 2024-11-14T14:00:40,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840403) roll requested 2024-11-14T14:00:40,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840412 2024-11-14T14:00:40,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840412 2024-11-14T14:00:40,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840412) roll requested 2024-11-14T14:00:40,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840419 2024-11-14T14:00:40,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840419 2024-11-14T14:00:40,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840419) roll requested 2024-11-14T14:00:40,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840425 2024-11-14T14:00:40,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840425 2024-11-14T14:00:40,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840425) roll requested 2024-11-14T14:00:40,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840432 2024-11-14T14:00:40,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840432 2024-11-14T14:00:40,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840432) roll requested 2024-11-14T14:00:40,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840438 2024-11-14T14:00:40,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840438 2024-11-14T14:00:40,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840438) roll requested 2024-11-14T14:00:40,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840445 2024-11-14T14:00:40,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840445 2024-11-14T14:00:40,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840445) roll requested 2024-11-14T14:00:40,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840452 2024-11-14T14:00:40,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840452 2024-11-14T14:00:40,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840452) roll requested 2024-11-14T14:00:40,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840458 2024-11-14T14:00:40,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840458 2024-11-14T14:00:40,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840458) roll requested 2024-11-14T14:00:40,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840465 2024-11-14T14:00:40,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840465 2024-11-14T14:00:40,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840465) roll requested 2024-11-14T14:00:40,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840471 2024-11-14T14:00:40,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840471 2024-11-14T14:00:40,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840471) roll requested 2024-11-14T14:00:40,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840478 2024-11-14T14:00:40,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840478 2024-11-14T14:00:40,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840478) roll requested 2024-11-14T14:00:40,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840485 2024-11-14T14:00:40,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840485 2024-11-14T14:00:40,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840485) roll requested 2024-11-14T14:00:40,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840491 2024-11-14T14:00:40,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840491 2024-11-14T14:00:40,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840491) roll requested 2024-11-14T14:00:40,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840497 2024-11-14T14:00:40,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840497 2024-11-14T14:00:40,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840497) roll requested 2024-11-14T14:00:40,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840503 2024-11-14T14:00:40,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840503 2024-11-14T14:00:40,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840503) roll requested 2024-11-14T14:00:40,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840510 2024-11-14T14:00:40,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840510 2024-11-14T14:00:40,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840510) roll requested 2024-11-14T14:00:40,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840516 2024-11-14T14:00:40,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840516 2024-11-14T14:00:40,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840516) roll requested 2024-11-14T14:00:40,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840522 2024-11-14T14:00:40,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840522 2024-11-14T14:00:40,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840522) roll requested 2024-11-14T14:00:40,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840528 2024-11-14T14:00:40,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840528 2024-11-14T14:00:40,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840528) roll requested 2024-11-14T14:00:40,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840535 2024-11-14T14:00:40,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840535 2024-11-14T14:00:40,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840535) roll requested 2024-11-14T14:00:40,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840541 2024-11-14T14:00:40,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840541 2024-11-14T14:00:40,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840541) roll requested 2024-11-14T14:00:40,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840547 2024-11-14T14:00:40,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840547 2024-11-14T14:00:40,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840547) roll requested 2024-11-14T14:00:40,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840553 2024-11-14T14:00:40,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840553 2024-11-14T14:00:40,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840553) roll requested 2024-11-14T14:00:40,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840559 2024-11-14T14:00:40,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840559 2024-11-14T14:00:40,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840559) roll requested 2024-11-14T14:00:40,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840566 2024-11-14T14:00:40,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840566 2024-11-14T14:00:40,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840566) roll requested 2024-11-14T14:00:40,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840572 2024-11-14T14:00:40,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840572 2024-11-14T14:00:40,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840572) roll requested 2024-11-14T14:00:40,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840578 2024-11-14T14:00:40,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840578 2024-11-14T14:00:40,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840578) roll requested 2024-11-14T14:00:40,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840584 2024-11-14T14:00:40,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840584 2024-11-14T14:00:40,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840584) roll requested 2024-11-14T14:00:40,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840590 2024-11-14T14:00:40,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840590 2024-11-14T14:00:40,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840590) roll requested 2024-11-14T14:00:40,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840596 2024-11-14T14:00:40,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840596 2024-11-14T14:00:40,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840596) roll requested 2024-11-14T14:00:40,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840603 2024-11-14T14:00:40,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840603 2024-11-14T14:00:40,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840603) roll requested 2024-11-14T14:00:40,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840609 2024-11-14T14:00:40,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840609 2024-11-14T14:00:40,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840609) roll requested 2024-11-14T14:00:40,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840615 2024-11-14T14:00:40,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840615 2024-11-14T14:00:40,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840615) roll requested 2024-11-14T14:00:40,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840621 2024-11-14T14:00:40,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840621 2024-11-14T14:00:40,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840621) roll requested 2024-11-14T14:00:40,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840628 2024-11-14T14:00:40,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840628 2024-11-14T14:00:40,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840628) roll requested 2024-11-14T14:00:40,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840634 2024-11-14T14:00:40,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840634 2024-11-14T14:00:40,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840634) roll requested 2024-11-14T14:00:40,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840640 2024-11-14T14:00:40,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840640 2024-11-14T14:00:40,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840640) roll requested 2024-11-14T14:00:40,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840646 2024-11-14T14:00:40,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840646 2024-11-14T14:00:40,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840646) roll requested 2024-11-14T14:00:40,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840652 2024-11-14T14:00:40,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840652 2024-11-14T14:00:40,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840652) roll requested 2024-11-14T14:00:40,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840664 2024-11-14T14:00:40,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840664 2024-11-14T14:00:40,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840664) roll requested 2024-11-14T14:00:40,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840670 2024-11-14T14:00:40,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840670 2024-11-14T14:00:40,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840670) roll requested 2024-11-14T14:00:40,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840676 2024-11-14T14:00:40,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840676 2024-11-14T14:00:40,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840676) roll requested 2024-11-14T14:00:40,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840683 2024-11-14T14:00:40,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840683 2024-11-14T14:00:40,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840683) roll requested 2024-11-14T14:00:40,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840689 2024-11-14T14:00:40,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840689 2024-11-14T14:00:40,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840689) roll requested 2024-11-14T14:00:40,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840696 2024-11-14T14:00:40,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840696 2024-11-14T14:00:40,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840696) roll requested 2024-11-14T14:00:40,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840702 2024-11-14T14:00:40,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840702 2024-11-14T14:00:40,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840702) roll requested 2024-11-14T14:00:40,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840708 2024-11-14T14:00:40,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840708 2024-11-14T14:00:40,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840708) roll requested 2024-11-14T14:00:40,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840714 2024-11-14T14:00:40,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840714 2024-11-14T14:00:40,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840714) roll requested 2024-11-14T14:00:40,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840721 2024-11-14T14:00:40,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840721 2024-11-14T14:00:40,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840721) roll requested 2024-11-14T14:00:40,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840730 2024-11-14T14:00:40,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840730 2024-11-14T14:00:40,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840730) roll requested 2024-11-14T14:00:40,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840740 2024-11-14T14:00:40,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840740 2024-11-14T14:00:40,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840740) roll requested 2024-11-14T14:00:40,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840746 2024-11-14T14:00:40,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840746 2024-11-14T14:00:40,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840746) roll requested 2024-11-14T14:00:40,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840752 2024-11-14T14:00:40,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840752 2024-11-14T14:00:40,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840752) roll requested 2024-11-14T14:00:40,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840758 2024-11-14T14:00:40,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840758 2024-11-14T14:00:40,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840758) roll requested 2024-11-14T14:00:40,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840765 2024-11-14T14:00:40,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840765 2024-11-14T14:00:40,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840765) roll requested 2024-11-14T14:00:40,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840772 2024-11-14T14:00:40,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840772 2024-11-14T14:00:40,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840772) roll requested 2024-11-14T14:00:40,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840778 2024-11-14T14:00:40,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840778 2024-11-14T14:00:40,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840778) roll requested 2024-11-14T14:00:40,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840784 2024-11-14T14:00:40,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840784 2024-11-14T14:00:40,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840784) roll requested 2024-11-14T14:00:40,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840790 2024-11-14T14:00:40,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840790 2024-11-14T14:00:40,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840790) roll requested 2024-11-14T14:00:40,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840796 2024-11-14T14:00:40,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840796 2024-11-14T14:00:40,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840796) roll requested 2024-11-14T14:00:40,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840803 2024-11-14T14:00:40,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840803 2024-11-14T14:00:40,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840803) roll requested 2024-11-14T14:00:40,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840809 2024-11-14T14:00:40,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840809 2024-11-14T14:00:40,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840809) roll requested 2024-11-14T14:00:40,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840815 2024-11-14T14:00:40,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840815 2024-11-14T14:00:40,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840815) roll requested 2024-11-14T14:00:40,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840822 2024-11-14T14:00:40,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840822 2024-11-14T14:00:40,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840822) roll requested 2024-11-14T14:00:40,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840827 2024-11-14T14:00:40,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840827 2024-11-14T14:00:40,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840827) roll requested 2024-11-14T14:00:40,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840834 2024-11-14T14:00:40,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840834 2024-11-14T14:00:40,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840834) roll requested 2024-11-14T14:00:40,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840840 2024-11-14T14:00:40,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840840 2024-11-14T14:00:40,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840840) roll requested 2024-11-14T14:00:40,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840846 2024-11-14T14:00:40,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840846 2024-11-14T14:00:40,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840846) roll requested 2024-11-14T14:00:40,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840852 2024-11-14T14:00:40,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840852 2024-11-14T14:00:40,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840852) roll requested 2024-11-14T14:00:40,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840858 2024-11-14T14:00:40,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840858 2024-11-14T14:00:40,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840858) roll requested 2024-11-14T14:00:40,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840864 2024-11-14T14:00:40,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840864 2024-11-14T14:00:40,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840864) roll requested 2024-11-14T14:00:40,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840870 2024-11-14T14:00:40,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840870 2024-11-14T14:00:40,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840870) roll requested 2024-11-14T14:00:40,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840879 2024-11-14T14:00:40,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840879 2024-11-14T14:00:40,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840879) roll requested 2024-11-14T14:00:40,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840889 2024-11-14T14:00:40,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840889 2024-11-14T14:00:40,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840889) roll requested 2024-11-14T14:00:40,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840895 2024-11-14T14:00:40,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840895 2024-11-14T14:00:40,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840895) roll requested 2024-11-14T14:00:40,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840905 2024-11-14T14:00:40,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840905 2024-11-14T14:00:40,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840905) roll requested 2024-11-14T14:00:40,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840911 2024-11-14T14:00:40,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840911 2024-11-14T14:00:40,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840911) roll requested 2024-11-14T14:00:40,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840918 2024-11-14T14:00:40,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840918 2024-11-14T14:00:40,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840918) roll requested 2024-11-14T14:00:40,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840924 2024-11-14T14:00:40,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840924 2024-11-14T14:00:40,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840924) roll requested 2024-11-14T14:00:40,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840931 2024-11-14T14:00:40,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840931 2024-11-14T14:00:40,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840931) roll requested 2024-11-14T14:00:40,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840937 2024-11-14T14:00:40,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840937 2024-11-14T14:00:40,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840937) roll requested 2024-11-14T14:00:40,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840944 2024-11-14T14:00:40,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840944 2024-11-14T14:00:40,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840944) roll requested 2024-11-14T14:00:40,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840950 2024-11-14T14:00:40,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840950 2024-11-14T14:00:40,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840950) roll requested 2024-11-14T14:00:40,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840956 2024-11-14T14:00:40,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840956 2024-11-14T14:00:40,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840956) roll requested 2024-11-14T14:00:40,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840962 2024-11-14T14:00:40,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840962 2024-11-14T14:00:40,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840962) roll requested 2024-11-14T14:00:40,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840968 2024-11-14T14:00:40,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840968 2024-11-14T14:00:40,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840968) roll requested 2024-11-14T14:00:40,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840975 2024-11-14T14:00:40,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840975 2024-11-14T14:00:40,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840975) roll requested 2024-11-14T14:00:40,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840981 2024-11-14T14:00:40,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840981 2024-11-14T14:00:40,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840981) roll requested 2024-11-14T14:00:40,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840988 2024-11-14T14:00:40,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:40,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840988 2024-11-14T14:00:40,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:40,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:40,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840988) roll requested 2024-11-14T14:00:40,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592840994 2024-11-14T14:00:41,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840994 2024-11-14T14:00:41,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592840994) roll requested 2024-11-14T14:00:41,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841000 2024-11-14T14:00:41,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592840994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841000 2024-11-14T14:00:41,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841000) roll requested 2024-11-14T14:00:41,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841007 2024-11-14T14:00:41,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841007 2024-11-14T14:00:41,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841007) roll requested 2024-11-14T14:00:41,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841014 2024-11-14T14:00:41,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841014 2024-11-14T14:00:41,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841014) roll requested 2024-11-14T14:00:41,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841020 2024-11-14T14:00:41,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841020 2024-11-14T14:00:41,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841020) roll requested 2024-11-14T14:00:41,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841026 2024-11-14T14:00:41,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841026 2024-11-14T14:00:41,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841026) roll requested 2024-11-14T14:00:41,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841034 2024-11-14T14:00:41,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841034 2024-11-14T14:00:41,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841034) roll requested 2024-11-14T14:00:41,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841040 2024-11-14T14:00:41,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841040 2024-11-14T14:00:41,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841040) roll requested 2024-11-14T14:00:41,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841046 2024-11-14T14:00:41,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841046 2024-11-14T14:00:41,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841046) roll requested 2024-11-14T14:00:41,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841052 2024-11-14T14:00:41,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841052 2024-11-14T14:00:41,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841052) roll requested 2024-11-14T14:00:41,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841059 2024-11-14T14:00:41,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841059 2024-11-14T14:00:41,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841059) roll requested 2024-11-14T14:00:41,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841065 2024-11-14T14:00:41,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841065 2024-11-14T14:00:41,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841065) roll requested 2024-11-14T14:00:41,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841072 2024-11-14T14:00:41,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841072 2024-11-14T14:00:41,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841072) roll requested 2024-11-14T14:00:41,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841078 2024-11-14T14:00:41,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841078 2024-11-14T14:00:41,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841078) roll requested 2024-11-14T14:00:41,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841085 2024-11-14T14:00:41,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841085 2024-11-14T14:00:41,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841085) roll requested 2024-11-14T14:00:41,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841091 2024-11-14T14:00:41,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841091 2024-11-14T14:00:41,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841091) roll requested 2024-11-14T14:00:41,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841097 2024-11-14T14:00:41,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841097 2024-11-14T14:00:41,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841097) roll requested 2024-11-14T14:00:41,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841103 2024-11-14T14:00:41,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841103 2024-11-14T14:00:41,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841103) roll requested 2024-11-14T14:00:41,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841110 2024-11-14T14:00:41,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841110 2024-11-14T14:00:41,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841110) roll requested 2024-11-14T14:00:41,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841116 2024-11-14T14:00:41,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841116 2024-11-14T14:00:41,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841116) roll requested 2024-11-14T14:00:41,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841122 2024-11-14T14:00:41,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841122 2024-11-14T14:00:41,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841122) roll requested 2024-11-14T14:00:41,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841129 2024-11-14T14:00:41,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841129 2024-11-14T14:00:41,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841129) roll requested 2024-11-14T14:00:41,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841140 2024-11-14T14:00:41,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841140 2024-11-14T14:00:41,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841140) roll requested 2024-11-14T14:00:41,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841148 2024-11-14T14:00:41,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841148 2024-11-14T14:00:41,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841148) roll requested 2024-11-14T14:00:41,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841156 2024-11-14T14:00:41,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841156 2024-11-14T14:00:41,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841156) roll requested 2024-11-14T14:00:41,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841162 2024-11-14T14:00:41,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841162 2024-11-14T14:00:41,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841162) roll requested 2024-11-14T14:00:41,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841168 2024-11-14T14:00:41,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841168 2024-11-14T14:00:41,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841168) roll requested 2024-11-14T14:00:41,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841174 2024-11-14T14:00:41,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841174 2024-11-14T14:00:41,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841174) roll requested 2024-11-14T14:00:41,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841180 2024-11-14T14:00:41,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841180 2024-11-14T14:00:41,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841180) roll requested 2024-11-14T14:00:41,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841187 2024-11-14T14:00:41,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841187 2024-11-14T14:00:41,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841187) roll requested 2024-11-14T14:00:41,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841195 2024-11-14T14:00:41,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841195 2024-11-14T14:00:41,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841195) roll requested 2024-11-14T14:00:41,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841202 2024-11-14T14:00:41,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841202 2024-11-14T14:00:41,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841202) roll requested 2024-11-14T14:00:41,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841209 2024-11-14T14:00:41,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841209 2024-11-14T14:00:41,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841209) roll requested 2024-11-14T14:00:41,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841215 2024-11-14T14:00:41,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841215 2024-11-14T14:00:41,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841215) roll requested 2024-11-14T14:00:41,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841221 2024-11-14T14:00:41,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841221 2024-11-14T14:00:41,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841221) roll requested 2024-11-14T14:00:41,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841228 2024-11-14T14:00:41,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841228 2024-11-14T14:00:41,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841228) roll requested 2024-11-14T14:00:41,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841234 2024-11-14T14:00:41,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841234 2024-11-14T14:00:41,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841234) roll requested 2024-11-14T14:00:41,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841240 2024-11-14T14:00:41,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841240 2024-11-14T14:00:41,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841240) roll requested 2024-11-14T14:00:41,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841246 2024-11-14T14:00:41,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841246 2024-11-14T14:00:41,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841246) roll requested 2024-11-14T14:00:41,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841255 2024-11-14T14:00:41,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841255 2024-11-14T14:00:41,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841255) roll requested 2024-11-14T14:00:41,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841262 2024-11-14T14:00:41,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841262 2024-11-14T14:00:41,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841262) roll requested 2024-11-14T14:00:41,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841269 2024-11-14T14:00:41,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841269 2024-11-14T14:00:41,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841269) roll requested 2024-11-14T14:00:41,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841275 2024-11-14T14:00:41,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841275 2024-11-14T14:00:41,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841275) roll requested 2024-11-14T14:00:41,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841281 2024-11-14T14:00:41,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841281 2024-11-14T14:00:41,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841281) roll requested 2024-11-14T14:00:41,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841287 2024-11-14T14:00:41,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841287 2024-11-14T14:00:41,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841287) roll requested 2024-11-14T14:00:41,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841295 2024-11-14T14:00:41,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841295 2024-11-14T14:00:41,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841295) roll requested 2024-11-14T14:00:41,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841301 2024-11-14T14:00:41,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841301 2024-11-14T14:00:41,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841301) roll requested 2024-11-14T14:00:41,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841308 2024-11-14T14:00:41,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841308 2024-11-14T14:00:41,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841308) roll requested 2024-11-14T14:00:41,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841314 2024-11-14T14:00:41,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841314 2024-11-14T14:00:41,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841314) roll requested 2024-11-14T14:00:41,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841324 2024-11-14T14:00:41,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841324 2024-11-14T14:00:41,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841324) roll requested 2024-11-14T14:00:41,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841333 2024-11-14T14:00:41,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841333 2024-11-14T14:00:41,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841333) roll requested 2024-11-14T14:00:41,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841341 2024-11-14T14:00:41,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841341 2024-11-14T14:00:41,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841341) roll requested 2024-11-14T14:00:41,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841348 2024-11-14T14:00:41,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841348 2024-11-14T14:00:41,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841348) roll requested 2024-11-14T14:00:41,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841356 2024-11-14T14:00:41,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841356 2024-11-14T14:00:41,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841356) roll requested 2024-11-14T14:00:41,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841362 2024-11-14T14:00:41,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841362 2024-11-14T14:00:41,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841362) roll requested 2024-11-14T14:00:41,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841368 2024-11-14T14:00:41,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841368 2024-11-14T14:00:41,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841368) roll requested 2024-11-14T14:00:41,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841377 2024-11-14T14:00:41,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841377 2024-11-14T14:00:41,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841377) roll requested 2024-11-14T14:00:41,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841384 2024-11-14T14:00:41,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841384 2024-11-14T14:00:41,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841384) roll requested 2024-11-14T14:00:41,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841390 2024-11-14T14:00:41,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841390 2024-11-14T14:00:41,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841390) roll requested 2024-11-14T14:00:41,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841397 2024-11-14T14:00:41,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841397 2024-11-14T14:00:41,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841397) roll requested 2024-11-14T14:00:41,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841403 2024-11-14T14:00:41,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841403 2024-11-14T14:00:41,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841403) roll requested 2024-11-14T14:00:41,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841410 2024-11-14T14:00:41,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841410 2024-11-14T14:00:41,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841410) roll requested 2024-11-14T14:00:41,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841416 2024-11-14T14:00:41,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841416 2024-11-14T14:00:41,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841416) roll requested 2024-11-14T14:00:41,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841422 2024-11-14T14:00:41,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841422 2024-11-14T14:00:41,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841422) roll requested 2024-11-14T14:00:41,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841428 2024-11-14T14:00:41,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841428 2024-11-14T14:00:41,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841428) roll requested 2024-11-14T14:00:41,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841435 2024-11-14T14:00:41,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841435 2024-11-14T14:00:41,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841435) roll requested 2024-11-14T14:00:41,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841441 2024-11-14T14:00:41,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841441 2024-11-14T14:00:41,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841441) roll requested 2024-11-14T14:00:41,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841447 2024-11-14T14:00:41,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841447 2024-11-14T14:00:41,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841447) roll requested 2024-11-14T14:00:41,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841453 2024-11-14T14:00:41,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841453 2024-11-14T14:00:41,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841453) roll requested 2024-11-14T14:00:41,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841459 2024-11-14T14:00:41,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841459 2024-11-14T14:00:41,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841459) roll requested 2024-11-14T14:00:41,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841465 2024-11-14T14:00:41,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841465 2024-11-14T14:00:41,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841465) roll requested 2024-11-14T14:00:41,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841471 2024-11-14T14:00:41,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841471 2024-11-14T14:00:41,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841471) roll requested 2024-11-14T14:00:41,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841477 2024-11-14T14:00:41,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841477 2024-11-14T14:00:41,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841477) roll requested 2024-11-14T14:00:41,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841483 2024-11-14T14:00:41,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841483 2024-11-14T14:00:41,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841483) roll requested 2024-11-14T14:00:41,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841490 2024-11-14T14:00:41,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841490 2024-11-14T14:00:41,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841490) roll requested 2024-11-14T14:00:41,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841496 2024-11-14T14:00:41,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841496 2024-11-14T14:00:41,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841496) roll requested 2024-11-14T14:00:41,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841502 2024-11-14T14:00:41,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841502 2024-11-14T14:00:41,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841502) roll requested 2024-11-14T14:00:41,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841508 2024-11-14T14:00:41,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841508 2024-11-14T14:00:41,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841508) roll requested 2024-11-14T14:00:41,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841514 2024-11-14T14:00:41,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841514 2024-11-14T14:00:41,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841514) roll requested 2024-11-14T14:00:41,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841520 2024-11-14T14:00:41,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841520 2024-11-14T14:00:41,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841520) roll requested 2024-11-14T14:00:41,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841527 2024-11-14T14:00:41,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841527 2024-11-14T14:00:41,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841527) roll requested 2024-11-14T14:00:41,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841533 2024-11-14T14:00:41,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841533 2024-11-14T14:00:41,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841533) roll requested 2024-11-14T14:00:41,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841539 2024-11-14T14:00:41,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841539 2024-11-14T14:00:41,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841539) roll requested 2024-11-14T14:00:41,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841546 2024-11-14T14:00:41,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841546 2024-11-14T14:00:41,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841546) roll requested 2024-11-14T14:00:41,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841552 2024-11-14T14:00:41,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841552 2024-11-14T14:00:41,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841552) roll requested 2024-11-14T14:00:41,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841558 2024-11-14T14:00:41,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841558 2024-11-14T14:00:41,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841558) roll requested 2024-11-14T14:00:41,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841564 2024-11-14T14:00:41,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841564 2024-11-14T14:00:41,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841564) roll requested 2024-11-14T14:00:41,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841570 2024-11-14T14:00:41,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841570 2024-11-14T14:00:41,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841570) roll requested 2024-11-14T14:00:41,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841576 2024-11-14T14:00:41,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841576 2024-11-14T14:00:41,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841576) roll requested 2024-11-14T14:00:41,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841583 2024-11-14T14:00:41,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841583 2024-11-14T14:00:41,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841583) roll requested 2024-11-14T14:00:41,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841589 2024-11-14T14:00:41,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841589 2024-11-14T14:00:41,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841589) roll requested 2024-11-14T14:00:41,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841595 2024-11-14T14:00:41,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841595 2024-11-14T14:00:41,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841595) roll requested 2024-11-14T14:00:41,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841602 2024-11-14T14:00:41,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841602 2024-11-14T14:00:41,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841602) roll requested 2024-11-14T14:00:41,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841608 2024-11-14T14:00:41,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841608 2024-11-14T14:00:41,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841608) roll requested 2024-11-14T14:00:41,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841614 2024-11-14T14:00:41,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841614 2024-11-14T14:00:41,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841614) roll requested 2024-11-14T14:00:41,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841620 2024-11-14T14:00:41,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841620 2024-11-14T14:00:41,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841620) roll requested 2024-11-14T14:00:41,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841626 2024-11-14T14:00:41,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841626 2024-11-14T14:00:41,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841626) roll requested 2024-11-14T14:00:41,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841633 2024-11-14T14:00:41,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841633 2024-11-14T14:00:41,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841633) roll requested 2024-11-14T14:00:41,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841639 2024-11-14T14:00:41,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841639 2024-11-14T14:00:41,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841639) roll requested 2024-11-14T14:00:41,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841645 2024-11-14T14:00:41,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841645 2024-11-14T14:00:41,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841645) roll requested 2024-11-14T14:00:41,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841652 2024-11-14T14:00:41,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841652 2024-11-14T14:00:41,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841652) roll requested 2024-11-14T14:00:41,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841658 2024-11-14T14:00:41,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841658 2024-11-14T14:00:41,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841658) roll requested 2024-11-14T14:00:41,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841664 2024-11-14T14:00:41,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841664 2024-11-14T14:00:41,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841664) roll requested 2024-11-14T14:00:41,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841670 2024-11-14T14:00:41,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841670 2024-11-14T14:00:41,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841670) roll requested 2024-11-14T14:00:41,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841676 2024-11-14T14:00:41,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841676 2024-11-14T14:00:41,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841676) roll requested 2024-11-14T14:00:41,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841683 2024-11-14T14:00:41,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841683 2024-11-14T14:00:41,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841683) roll requested 2024-11-14T14:00:41,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841689 2024-11-14T14:00:41,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841689 2024-11-14T14:00:41,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841689) roll requested 2024-11-14T14:00:41,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841695 2024-11-14T14:00:41,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841695 2024-11-14T14:00:41,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841695) roll requested 2024-11-14T14:00:41,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841701 2024-11-14T14:00:41,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841701 2024-11-14T14:00:41,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841701) roll requested 2024-11-14T14:00:41,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841707 2024-11-14T14:00:41,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841707 2024-11-14T14:00:41,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841707) roll requested 2024-11-14T14:00:41,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841713 2024-11-14T14:00:41,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841713 2024-11-14T14:00:41,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841713) roll requested 2024-11-14T14:00:41,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841719 2024-11-14T14:00:41,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841719 2024-11-14T14:00:41,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841719) roll requested 2024-11-14T14:00:41,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841726 2024-11-14T14:00:41,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841726 2024-11-14T14:00:41,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841726) roll requested 2024-11-14T14:00:41,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841733 2024-11-14T14:00:41,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841733 2024-11-14T14:00:41,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841733) roll requested 2024-11-14T14:00:41,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841739 2024-11-14T14:00:41,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841739 2024-11-14T14:00:41,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841739) roll requested 2024-11-14T14:00:41,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841745 2024-11-14T14:00:41,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841745 2024-11-14T14:00:41,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841745) roll requested 2024-11-14T14:00:41,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841751 2024-11-14T14:00:41,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841751 2024-11-14T14:00:41,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841751) roll requested 2024-11-14T14:00:41,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841757 2024-11-14T14:00:41,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841757 2024-11-14T14:00:41,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841757) roll requested 2024-11-14T14:00:41,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841764 2024-11-14T14:00:41,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841764 2024-11-14T14:00:41,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841764) roll requested 2024-11-14T14:00:41,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841770 2024-11-14T14:00:41,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841770 2024-11-14T14:00:41,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841770) roll requested 2024-11-14T14:00:41,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841775 2024-11-14T14:00:41,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841775 2024-11-14T14:00:41,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841775) roll requested 2024-11-14T14:00:41,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841784 2024-11-14T14:00:41,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841784 2024-11-14T14:00:41,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841784) roll requested 2024-11-14T14:00:41,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841796 2024-11-14T14:00:41,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841796 2024-11-14T14:00:41,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841796) roll requested 2024-11-14T14:00:41,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841805 2024-11-14T14:00:41,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841805 2024-11-14T14:00:41,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841805) roll requested 2024-11-14T14:00:41,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841813 2024-11-14T14:00:41,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841813 2024-11-14T14:00:41,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841813) roll requested 2024-11-14T14:00:41,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841819 2024-11-14T14:00:41,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841819 2024-11-14T14:00:41,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841819) roll requested 2024-11-14T14:00:41,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841825 2024-11-14T14:00:41,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841825 2024-11-14T14:00:41,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841825) roll requested 2024-11-14T14:00:41,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841835 2024-11-14T14:00:41,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841835 2024-11-14T14:00:41,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841835) roll requested 2024-11-14T14:00:41,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841841 2024-11-14T14:00:41,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841841 2024-11-14T14:00:41,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841841) roll requested 2024-11-14T14:00:41,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841847 2024-11-14T14:00:41,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841847 2024-11-14T14:00:41,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841847) roll requested 2024-11-14T14:00:41,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841853 2024-11-14T14:00:41,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841853 2024-11-14T14:00:41,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841853) roll requested 2024-11-14T14:00:41,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841861 2024-11-14T14:00:41,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841861 2024-11-14T14:00:41,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841861) roll requested 2024-11-14T14:00:41,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841867 2024-11-14T14:00:41,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841867 2024-11-14T14:00:41,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841867) roll requested 2024-11-14T14:00:41,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841873 2024-11-14T14:00:41,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841873 2024-11-14T14:00:41,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841873) roll requested 2024-11-14T14:00:41,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841879 2024-11-14T14:00:41,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841879 2024-11-14T14:00:41,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841879) roll requested 2024-11-14T14:00:41,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841887 2024-11-14T14:00:41,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841887 2024-11-14T14:00:41,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841887) roll requested 2024-11-14T14:00:41,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841893 2024-11-14T14:00:41,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841893 2024-11-14T14:00:41,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841893) roll requested 2024-11-14T14:00:41,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841900 2024-11-14T14:00:41,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841900 2024-11-14T14:00:41,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841900) roll requested 2024-11-14T14:00:41,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841907 2024-11-14T14:00:41,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841907 2024-11-14T14:00:41,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841907) roll requested 2024-11-14T14:00:41,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841914 2024-11-14T14:00:41,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841914 2024-11-14T14:00:41,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841914) roll requested 2024-11-14T14:00:41,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841921 2024-11-14T14:00:41,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841921 2024-11-14T14:00:41,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841921) roll requested 2024-11-14T14:00:41,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841929 2024-11-14T14:00:41,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841929 2024-11-14T14:00:41,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841929) roll requested 2024-11-14T14:00:41,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841935 2024-11-14T14:00:41,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841935 2024-11-14T14:00:41,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841935) roll requested 2024-11-14T14:00:41,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841941 2024-11-14T14:00:41,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841941 2024-11-14T14:00:41,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841941) roll requested 2024-11-14T14:00:41,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841948 2024-11-14T14:00:41,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841948 2024-11-14T14:00:41,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841948) roll requested 2024-11-14T14:00:41,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841955 2024-11-14T14:00:41,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841955 2024-11-14T14:00:41,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841955) roll requested 2024-11-14T14:00:41,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841962 2024-11-14T14:00:41,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841962 2024-11-14T14:00:41,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841962) roll requested 2024-11-14T14:00:41,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841969 2024-11-14T14:00:41,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841969 2024-11-14T14:00:41,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841969) roll requested 2024-11-14T14:00:41,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841976 2024-11-14T14:00:41,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841976 2024-11-14T14:00:41,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841976) roll requested 2024-11-14T14:00:41,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841982 2024-11-14T14:00:41,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841982 2024-11-14T14:00:41,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841982) roll requested 2024-11-14T14:00:41,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841989 2024-11-14T14:00:41,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:41,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841989 2024-11-14T14:00:41,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:41,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:41,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841989) roll requested 2024-11-14T14:00:41,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592841995 2024-11-14T14:00:42,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841995 2024-11-14T14:00:42,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592841995) roll requested 2024-11-14T14:00:42,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842002 2024-11-14T14:00:42,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592841995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842002 2024-11-14T14:00:42,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842002) roll requested 2024-11-14T14:00:42,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842008 2024-11-14T14:00:42,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842008 2024-11-14T14:00:42,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842008) roll requested 2024-11-14T14:00:42,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842015 2024-11-14T14:00:42,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842015 2024-11-14T14:00:42,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842015) roll requested 2024-11-14T14:00:42,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842022 2024-11-14T14:00:42,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842022 2024-11-14T14:00:42,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842022) roll requested 2024-11-14T14:00:42,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842029 2024-11-14T14:00:42,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842029 2024-11-14T14:00:42,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842029) roll requested 2024-11-14T14:00:42,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842036 2024-11-14T14:00:42,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842036 2024-11-14T14:00:42,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842036) roll requested 2024-11-14T14:00:42,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842043 2024-11-14T14:00:42,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842043 2024-11-14T14:00:42,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842043) roll requested 2024-11-14T14:00:42,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842051 2024-11-14T14:00:42,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842051 2024-11-14T14:00:42,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842051) roll requested 2024-11-14T14:00:42,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842059 2024-11-14T14:00:42,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842059 2024-11-14T14:00:42,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842059) roll requested 2024-11-14T14:00:42,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842065 2024-11-14T14:00:42,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842065 2024-11-14T14:00:42,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842065) roll requested 2024-11-14T14:00:42,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842072 2024-11-14T14:00:42,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842072 2024-11-14T14:00:42,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842072) roll requested 2024-11-14T14:00:42,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842079 2024-11-14T14:00:42,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842079 2024-11-14T14:00:42,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842079) roll requested 2024-11-14T14:00:42,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842087 2024-11-14T14:00:42,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842087 2024-11-14T14:00:42,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842087) roll requested 2024-11-14T14:00:42,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842094 2024-11-14T14:00:42,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842094 2024-11-14T14:00:42,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842094) roll requested 2024-11-14T14:00:42,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842102 2024-11-14T14:00:42,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842102 2024-11-14T14:00:42,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842102) roll requested 2024-11-14T14:00:42,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842109 2024-11-14T14:00:42,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842109 2024-11-14T14:00:42,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842109) roll requested 2024-11-14T14:00:42,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842117 2024-11-14T14:00:42,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842117 2024-11-14T14:00:42,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842117) roll requested 2024-11-14T14:00:42,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842124 2024-11-14T14:00:42,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842124 2024-11-14T14:00:42,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842124) roll requested 2024-11-14T14:00:42,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842131 2024-11-14T14:00:42,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842131 2024-11-14T14:00:42,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842131) roll requested 2024-11-14T14:00:42,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842139 2024-11-14T14:00:42,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842139 2024-11-14T14:00:42,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842139) roll requested 2024-11-14T14:00:42,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842147 2024-11-14T14:00:42,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842147 2024-11-14T14:00:42,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842147) roll requested 2024-11-14T14:00:42,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842154 2024-11-14T14:00:42,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842154 2024-11-14T14:00:42,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842154) roll requested 2024-11-14T14:00:42,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842162 2024-11-14T14:00:42,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842162 2024-11-14T14:00:42,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842162) roll requested 2024-11-14T14:00:42,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842168 2024-11-14T14:00:42,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842168 2024-11-14T14:00:42,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842168) roll requested 2024-11-14T14:00:42,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842176 2024-11-14T14:00:42,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842176 2024-11-14T14:00:42,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842176) roll requested 2024-11-14T14:00:42,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842183 2024-11-14T14:00:42,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842183 2024-11-14T14:00:42,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842183) roll requested 2024-11-14T14:00:42,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842190 2024-11-14T14:00:42,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842190 2024-11-14T14:00:42,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842190) roll requested 2024-11-14T14:00:42,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842197 2024-11-14T14:00:42,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842197 2024-11-14T14:00:42,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842197) roll requested 2024-11-14T14:00:42,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842205 2024-11-14T14:00:42,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842205 2024-11-14T14:00:42,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842205) roll requested 2024-11-14T14:00:42,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842213 2024-11-14T14:00:42,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842213 2024-11-14T14:00:42,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842213) roll requested 2024-11-14T14:00:42,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842220 2024-11-14T14:00:42,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842220 2024-11-14T14:00:42,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842220) roll requested 2024-11-14T14:00:42,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842227 2024-11-14T14:00:42,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842227 2024-11-14T14:00:42,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842227) roll requested 2024-11-14T14:00:42,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842234 2024-11-14T14:00:42,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842234 2024-11-14T14:00:42,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842234) roll requested 2024-11-14T14:00:42,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842241 2024-11-14T14:00:42,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842241 2024-11-14T14:00:42,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842241) roll requested 2024-11-14T14:00:42,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842248 2024-11-14T14:00:42,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842248 2024-11-14T14:00:42,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842248) roll requested 2024-11-14T14:00:42,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842255 2024-11-14T14:00:42,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842255 2024-11-14T14:00:42,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842255) roll requested 2024-11-14T14:00:42,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842262 2024-11-14T14:00:42,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842262 2024-11-14T14:00:42,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842262) roll requested 2024-11-14T14:00:42,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842269 2024-11-14T14:00:42,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842269 2024-11-14T14:00:42,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842269) roll requested 2024-11-14T14:00:42,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842279 2024-11-14T14:00:42,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842279 2024-11-14T14:00:42,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842279) roll requested 2024-11-14T14:00:42,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842286 2024-11-14T14:00:42,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842286 2024-11-14T14:00:42,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842286) roll requested 2024-11-14T14:00:42,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842293 2024-11-14T14:00:42,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842293 2024-11-14T14:00:42,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842293) roll requested 2024-11-14T14:00:42,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842303 2024-11-14T14:00:42,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842303 2024-11-14T14:00:42,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842303) roll requested 2024-11-14T14:00:42,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842310 2024-11-14T14:00:42,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842310 2024-11-14T14:00:42,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842310) roll requested 2024-11-14T14:00:42,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842317 2024-11-14T14:00:42,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842317 2024-11-14T14:00:42,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842317) roll requested 2024-11-14T14:00:42,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842324 2024-11-14T14:00:42,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842324 2024-11-14T14:00:42,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842324) roll requested 2024-11-14T14:00:42,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842331 2024-11-14T14:00:42,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842331 2024-11-14T14:00:42,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842331) roll requested 2024-11-14T14:00:42,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842338 2024-11-14T14:00:42,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842338 2024-11-14T14:00:42,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842338) roll requested 2024-11-14T14:00:42,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842345 2024-11-14T14:00:42,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842345 2024-11-14T14:00:42,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842345) roll requested 2024-11-14T14:00:42,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842352 2024-11-14T14:00:42,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842352 2024-11-14T14:00:42,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842352) roll requested 2024-11-14T14:00:42,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842358 2024-11-14T14:00:42,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842358 2024-11-14T14:00:42,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842358) roll requested 2024-11-14T14:00:42,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842365 2024-11-14T14:00:42,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842365 2024-11-14T14:00:42,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842365) roll requested 2024-11-14T14:00:42,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842371 2024-11-14T14:00:42,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842371 2024-11-14T14:00:42,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842371) roll requested 2024-11-14T14:00:42,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842379 2024-11-14T14:00:42,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842379 2024-11-14T14:00:42,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842379) roll requested 2024-11-14T14:00:42,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842386 2024-11-14T14:00:42,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842386 2024-11-14T14:00:42,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842386) roll requested 2024-11-14T14:00:42,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842393 2024-11-14T14:00:42,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842393 2024-11-14T14:00:42,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842393) roll requested 2024-11-14T14:00:42,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842400 2024-11-14T14:00:42,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842400 2024-11-14T14:00:42,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842400) roll requested 2024-11-14T14:00:42,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842407 2024-11-14T14:00:42,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842407 2024-11-14T14:00:42,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842407) roll requested 2024-11-14T14:00:42,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842414 2024-11-14T14:00:42,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842414 2024-11-14T14:00:42,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842414) roll requested 2024-11-14T14:00:42,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842423 2024-11-14T14:00:42,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842423 2024-11-14T14:00:42,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842423) roll requested 2024-11-14T14:00:42,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842435 2024-11-14T14:00:42,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842435 2024-11-14T14:00:42,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842435) roll requested 2024-11-14T14:00:42,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842443 2024-11-14T14:00:42,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842443 2024-11-14T14:00:42,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842443) roll requested 2024-11-14T14:00:42,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842449 2024-11-14T14:00:42,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842449 2024-11-14T14:00:42,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842449) roll requested 2024-11-14T14:00:42,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842461 2024-11-14T14:00:42,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842461 2024-11-14T14:00:42,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842461) roll requested 2024-11-14T14:00:42,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842467 2024-11-14T14:00:42,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842467 2024-11-14T14:00:42,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842467) roll requested 2024-11-14T14:00:42,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842474 2024-11-14T14:00:42,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842474 2024-11-14T14:00:42,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842474) roll requested 2024-11-14T14:00:42,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842481 2024-11-14T14:00:42,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842481 2024-11-14T14:00:42,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842481) roll requested 2024-11-14T14:00:42,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842487 2024-11-14T14:00:42,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842487 2024-11-14T14:00:42,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842487) roll requested 2024-11-14T14:00:42,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842494 2024-11-14T14:00:42,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842494 2024-11-14T14:00:42,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842494) roll requested 2024-11-14T14:00:42,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842501 2024-11-14T14:00:42,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842501 2024-11-14T14:00:42,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842501) roll requested 2024-11-14T14:00:42,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842508 2024-11-14T14:00:42,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842508 2024-11-14T14:00:42,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842508) roll requested 2024-11-14T14:00:42,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842515 2024-11-14T14:00:42,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842515 2024-11-14T14:00:42,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842515) roll requested 2024-11-14T14:00:42,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842522 2024-11-14T14:00:42,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842522 2024-11-14T14:00:42,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842522) roll requested 2024-11-14T14:00:42,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842529 2024-11-14T14:00:42,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842529 2024-11-14T14:00:42,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842529) roll requested 2024-11-14T14:00:42,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842536 2024-11-14T14:00:42,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842536 2024-11-14T14:00:42,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842536) roll requested 2024-11-14T14:00:42,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842543 2024-11-14T14:00:42,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842543 2024-11-14T14:00:42,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842543) roll requested 2024-11-14T14:00:42,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842550 2024-11-14T14:00:42,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842550 2024-11-14T14:00:42,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842550) roll requested 2024-11-14T14:00:42,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842557 2024-11-14T14:00:42,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842557 2024-11-14T14:00:42,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842557) roll requested 2024-11-14T14:00:42,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842563 2024-11-14T14:00:42,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842563 2024-11-14T14:00:42,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842563) roll requested 2024-11-14T14:00:42,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842570 2024-11-14T14:00:42,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842570 2024-11-14T14:00:42,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842570) roll requested 2024-11-14T14:00:42,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842578 2024-11-14T14:00:42,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842578 2024-11-14T14:00:42,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842578) roll requested 2024-11-14T14:00:42,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842585 2024-11-14T14:00:42,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842585 2024-11-14T14:00:42,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842585) roll requested 2024-11-14T14:00:42,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842593 2024-11-14T14:00:42,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842593 2024-11-14T14:00:42,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842593) roll requested 2024-11-14T14:00:42,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842600 2024-11-14T14:00:42,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842600 2024-11-14T14:00:42,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842600) roll requested 2024-11-14T14:00:42,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842607 2024-11-14T14:00:42,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842607 2024-11-14T14:00:42,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842607) roll requested 2024-11-14T14:00:42,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842614 2024-11-14T14:00:42,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842614 2024-11-14T14:00:42,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842614) roll requested 2024-11-14T14:00:42,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842622 2024-11-14T14:00:42,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842622 2024-11-14T14:00:42,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842622) roll requested 2024-11-14T14:00:42,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842631 2024-11-14T14:00:42,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842631 2024-11-14T14:00:42,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842631) roll requested 2024-11-14T14:00:42,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842638 2024-11-14T14:00:42,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842638 2024-11-14T14:00:42,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842638) roll requested 2024-11-14T14:00:42,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842644 2024-11-14T14:00:42,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842644 2024-11-14T14:00:42,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842644) roll requested 2024-11-14T14:00:42,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842650 2024-11-14T14:00:42,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842650 2024-11-14T14:00:42,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842650) roll requested 2024-11-14T14:00:42,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842656 2024-11-14T14:00:42,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842656 2024-11-14T14:00:42,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842656) roll requested 2024-11-14T14:00:42,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842663 2024-11-14T14:00:42,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842663 2024-11-14T14:00:42,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842663) roll requested 2024-11-14T14:00:42,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842670 2024-11-14T14:00:42,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842670 2024-11-14T14:00:42,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842670) roll requested 2024-11-14T14:00:42,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842677 2024-11-14T14:00:42,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842677 2024-11-14T14:00:42,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842677) roll requested 2024-11-14T14:00:42,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842684 2024-11-14T14:00:42,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842684 2024-11-14T14:00:42,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842684) roll requested 2024-11-14T14:00:42,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842691 2024-11-14T14:00:42,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842691 2024-11-14T14:00:42,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842691) roll requested 2024-11-14T14:00:42,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842698 2024-11-14T14:00:42,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842698 2024-11-14T14:00:42,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842698) roll requested 2024-11-14T14:00:42,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842705 2024-11-14T14:00:42,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842705 2024-11-14T14:00:42,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842705) roll requested 2024-11-14T14:00:42,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842713 2024-11-14T14:00:42,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842713 2024-11-14T14:00:42,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842713) roll requested 2024-11-14T14:00:42,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842720 2024-11-14T14:00:42,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842720 2024-11-14T14:00:42,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842720) roll requested 2024-11-14T14:00:42,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842727 2024-11-14T14:00:42,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842727 2024-11-14T14:00:42,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842727) roll requested 2024-11-14T14:00:42,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842733 2024-11-14T14:00:42,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842733 2024-11-14T14:00:42,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842733) roll requested 2024-11-14T14:00:42,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842739 2024-11-14T14:00:42,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842739 2024-11-14T14:00:42,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842739) roll requested 2024-11-14T14:00:42,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842748 2024-11-14T14:00:42,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842748 2024-11-14T14:00:42,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842748) roll requested 2024-11-14T14:00:42,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842756 2024-11-14T14:00:42,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842756 2024-11-14T14:00:42,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842756) roll requested 2024-11-14T14:00:42,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842763 2024-11-14T14:00:42,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842763 2024-11-14T14:00:42,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842763) roll requested 2024-11-14T14:00:42,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842771 2024-11-14T14:00:42,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842771 2024-11-14T14:00:42,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842771) roll requested 2024-11-14T14:00:42,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842778 2024-11-14T14:00:42,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842778 2024-11-14T14:00:42,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842778) roll requested 2024-11-14T14:00:42,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842785 2024-11-14T14:00:42,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842785 2024-11-14T14:00:42,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842785) roll requested 2024-11-14T14:00:42,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842792 2024-11-14T14:00:42,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842792 2024-11-14T14:00:42,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842792) roll requested 2024-11-14T14:00:42,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842799 2024-11-14T14:00:42,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842799 2024-11-14T14:00:42,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842799) roll requested 2024-11-14T14:00:42,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842807 2024-11-14T14:00:42,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842807 2024-11-14T14:00:42,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842807) roll requested 2024-11-14T14:00:42,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842815 2024-11-14T14:00:42,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842815 2024-11-14T14:00:42,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842815) roll requested 2024-11-14T14:00:42,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842822 2024-11-14T14:00:42,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842822 2024-11-14T14:00:42,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842822) roll requested 2024-11-14T14:00:42,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842829 2024-11-14T14:00:42,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842829 2024-11-14T14:00:42,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842829) roll requested 2024-11-14T14:00:42,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842836 2024-11-14T14:00:42,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842836 2024-11-14T14:00:42,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842836) roll requested 2024-11-14T14:00:42,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842843 2024-11-14T14:00:42,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842843 2024-11-14T14:00:42,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842843) roll requested 2024-11-14T14:00:42,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842850 2024-11-14T14:00:42,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842850 2024-11-14T14:00:42,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842850) roll requested 2024-11-14T14:00:42,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842858 2024-11-14T14:00:42,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842858 2024-11-14T14:00:42,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842858) roll requested 2024-11-14T14:00:42,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842865 2024-11-14T14:00:42,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842865 2024-11-14T14:00:42,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842865) roll requested 2024-11-14T14:00:42,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842873 2024-11-14T14:00:42,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842873 2024-11-14T14:00:42,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842873) roll requested 2024-11-14T14:00:42,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842880 2024-11-14T14:00:42,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842880 2024-11-14T14:00:42,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842880) roll requested 2024-11-14T14:00:42,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842887 2024-11-14T14:00:42,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842887 2024-11-14T14:00:42,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842887) roll requested 2024-11-14T14:00:42,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842894 2024-11-14T14:00:42,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842894 2024-11-14T14:00:42,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842894) roll requested 2024-11-14T14:00:42,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842900 2024-11-14T14:00:42,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842900 2024-11-14T14:00:42,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842900) roll requested 2024-11-14T14:00:42,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842906 2024-11-14T14:00:42,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842906 2024-11-14T14:00:42,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842906) roll requested 2024-11-14T14:00:42,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842912 2024-11-14T14:00:42,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842912 2024-11-14T14:00:42,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842912) roll requested 2024-11-14T14:00:42,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842918 2024-11-14T14:00:42,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842918 2024-11-14T14:00:42,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842918) roll requested 2024-11-14T14:00:42,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842925 2024-11-14T14:00:42,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842925 2024-11-14T14:00:42,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842925) roll requested 2024-11-14T14:00:42,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842931 2024-11-14T14:00:42,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842931 2024-11-14T14:00:42,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842931) roll requested 2024-11-14T14:00:42,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842937 2024-11-14T14:00:42,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842937 2024-11-14T14:00:42,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842937) roll requested 2024-11-14T14:00:42,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842943 2024-11-14T14:00:42,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842943 2024-11-14T14:00:42,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842943) roll requested 2024-11-14T14:00:42,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842950 2024-11-14T14:00:42,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842950 2024-11-14T14:00:42,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842950) roll requested 2024-11-14T14:00:42,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842958 2024-11-14T14:00:42,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842958 2024-11-14T14:00:42,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842958) roll requested 2024-11-14T14:00:42,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842965 2024-11-14T14:00:42,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842965 2024-11-14T14:00:42,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842965) roll requested 2024-11-14T14:00:42,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842972 2024-11-14T14:00:42,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842972 2024-11-14T14:00:42,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842972) roll requested 2024-11-14T14:00:42,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842980 2024-11-14T14:00:42,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842980 2024-11-14T14:00:42,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842980) roll requested 2024-11-14T14:00:42,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842987 2024-11-14T14:00:42,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:42,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842987 2024-11-14T14:00:42,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:42,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:42,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842987) roll requested 2024-11-14T14:00:42,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592842995 2024-11-14T14:00:43,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842995 2024-11-14T14:00:43,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592842995) roll requested 2024-11-14T14:00:43,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843002 2024-11-14T14:00:43,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592842995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843002 2024-11-14T14:00:43,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843002) roll requested 2024-11-14T14:00:43,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843009 2024-11-14T14:00:43,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843009 2024-11-14T14:00:43,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843009) roll requested 2024-11-14T14:00:43,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843017 2024-11-14T14:00:43,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843017 2024-11-14T14:00:43,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843017) roll requested 2024-11-14T14:00:43,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843025 2024-11-14T14:00:43,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843025 2024-11-14T14:00:43,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843025) roll requested 2024-11-14T14:00:43,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843032 2024-11-14T14:00:43,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843032 2024-11-14T14:00:43,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843032) roll requested 2024-11-14T14:00:43,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843039 2024-11-14T14:00:43,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843039 2024-11-14T14:00:43,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843039) roll requested 2024-11-14T14:00:43,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843046 2024-11-14T14:00:43,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843046 2024-11-14T14:00:43,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843046) roll requested 2024-11-14T14:00:43,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843053 2024-11-14T14:00:43,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843053 2024-11-14T14:00:43,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843053) roll requested 2024-11-14T14:00:43,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843060 2024-11-14T14:00:43,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843060 2024-11-14T14:00:43,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843060) roll requested 2024-11-14T14:00:43,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843067 2024-11-14T14:00:43,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843067 2024-11-14T14:00:43,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843067) roll requested 2024-11-14T14:00:43,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843074 2024-11-14T14:00:43,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843074 2024-11-14T14:00:43,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843074) roll requested 2024-11-14T14:00:43,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843082 2024-11-14T14:00:43,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843082 2024-11-14T14:00:43,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843082) roll requested 2024-11-14T14:00:43,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843089 2024-11-14T14:00:43,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843089 2024-11-14T14:00:43,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843089) roll requested 2024-11-14T14:00:43,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843096 2024-11-14T14:00:43,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843096 2024-11-14T14:00:43,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843096) roll requested 2024-11-14T14:00:43,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843102 2024-11-14T14:00:43,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843102 2024-11-14T14:00:43,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843102) roll requested 2024-11-14T14:00:43,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843110 2024-11-14T14:00:43,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843110 2024-11-14T14:00:43,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843110) roll requested 2024-11-14T14:00:43,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843117 2024-11-14T14:00:43,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843117 2024-11-14T14:00:43,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843117) roll requested 2024-11-14T14:00:43,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843132 2024-11-14T14:00:43,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843132 2024-11-14T14:00:43,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843132) roll requested 2024-11-14T14:00:43,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843141 2024-11-14T14:00:43,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843141 2024-11-14T14:00:43,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843141) roll requested 2024-11-14T14:00:43,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843149 2024-11-14T14:00:43,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843149 2024-11-14T14:00:43,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843149) roll requested 2024-11-14T14:00:43,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843156 2024-11-14T14:00:43,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843156 2024-11-14T14:00:43,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843156) roll requested 2024-11-14T14:00:43,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843164 2024-11-14T14:00:43,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843164 2024-11-14T14:00:43,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843164) roll requested 2024-11-14T14:00:43,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843171 2024-11-14T14:00:43,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843171 2024-11-14T14:00:43,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843171) roll requested 2024-11-14T14:00:43,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843179 2024-11-14T14:00:43,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843179 2024-11-14T14:00:43,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843179) roll requested 2024-11-14T14:00:43,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843187 2024-11-14T14:00:43,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843187 2024-11-14T14:00:43,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843187) roll requested 2024-11-14T14:00:43,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843195 2024-11-14T14:00:43,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843195 2024-11-14T14:00:43,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843195) roll requested 2024-11-14T14:00:43,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843203 2024-11-14T14:00:43,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843203 2024-11-14T14:00:43,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843203) roll requested 2024-11-14T14:00:43,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843211 2024-11-14T14:00:43,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843211 2024-11-14T14:00:43,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843211) roll requested 2024-11-14T14:00:43,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843218 2024-11-14T14:00:43,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843218 2024-11-14T14:00:43,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843218) roll requested 2024-11-14T14:00:43,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843226 2024-11-14T14:00:43,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843226 2024-11-14T14:00:43,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843226) roll requested 2024-11-14T14:00:43,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843233 2024-11-14T14:00:43,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843233 2024-11-14T14:00:43,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843233) roll requested 2024-11-14T14:00:43,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843241 2024-11-14T14:00:43,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843241 2024-11-14T14:00:43,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843241) roll requested 2024-11-14T14:00:43,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843249 2024-11-14T14:00:43,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843249 2024-11-14T14:00:43,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843249) roll requested 2024-11-14T14:00:43,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843256 2024-11-14T14:00:43,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843256 2024-11-14T14:00:43,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843256) roll requested 2024-11-14T14:00:43,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843263 2024-11-14T14:00:43,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843263 2024-11-14T14:00:43,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843263) roll requested 2024-11-14T14:00:43,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843271 2024-11-14T14:00:43,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843271 2024-11-14T14:00:43,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843271) roll requested 2024-11-14T14:00:43,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843278 2024-11-14T14:00:43,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843278 2024-11-14T14:00:43,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843278) roll requested 2024-11-14T14:00:43,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843285 2024-11-14T14:00:43,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843285 2024-11-14T14:00:43,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843285) roll requested 2024-11-14T14:00:43,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843293 2024-11-14T14:00:43,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843293 2024-11-14T14:00:43,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843293) roll requested 2024-11-14T14:00:43,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843302 2024-11-14T14:00:43,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843302 2024-11-14T14:00:43,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843302) roll requested 2024-11-14T14:00:43,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843309 2024-11-14T14:00:43,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843309 2024-11-14T14:00:43,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843309) roll requested 2024-11-14T14:00:43,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843316 2024-11-14T14:00:43,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843316 2024-11-14T14:00:43,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843316) roll requested 2024-11-14T14:00:43,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843324 2024-11-14T14:00:43,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843324 2024-11-14T14:00:43,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843324) roll requested 2024-11-14T14:00:43,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843331 2024-11-14T14:00:43,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843331 2024-11-14T14:00:43,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843331) roll requested 2024-11-14T14:00:43,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843339 2024-11-14T14:00:43,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843339 2024-11-14T14:00:43,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843339) roll requested 2024-11-14T14:00:43,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843347 2024-11-14T14:00:43,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843347 2024-11-14T14:00:43,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843347) roll requested 2024-11-14T14:00:43,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843355 2024-11-14T14:00:43,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843355 2024-11-14T14:00:43,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843355) roll requested 2024-11-14T14:00:43,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843363 2024-11-14T14:00:43,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843363 2024-11-14T14:00:43,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843363) roll requested 2024-11-14T14:00:43,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843370 2024-11-14T14:00:43,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843370 2024-11-14T14:00:43,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843370) roll requested 2024-11-14T14:00:43,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843377 2024-11-14T14:00:43,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843377 2024-11-14T14:00:43,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843377) roll requested 2024-11-14T14:00:43,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843385 2024-11-14T14:00:43,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843385 2024-11-14T14:00:43,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843385) roll requested 2024-11-14T14:00:43,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843392 2024-11-14T14:00:43,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843392 2024-11-14T14:00:43,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843392) roll requested 2024-11-14T14:00:43,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843399 2024-11-14T14:00:43,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843399 2024-11-14T14:00:43,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843399) roll requested 2024-11-14T14:00:43,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843407 2024-11-14T14:00:43,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843407 2024-11-14T14:00:43,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843407) roll requested 2024-11-14T14:00:43,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843415 2024-11-14T14:00:43,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843415 2024-11-14T14:00:43,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843415) roll requested 2024-11-14T14:00:43,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843422 2024-11-14T14:00:43,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843422 2024-11-14T14:00:43,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843422) roll requested 2024-11-14T14:00:43,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843430 2024-11-14T14:00:43,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843430 2024-11-14T14:00:43,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843430) roll requested 2024-11-14T14:00:43,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843437 2024-11-14T14:00:43,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843437 2024-11-14T14:00:43,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843437) roll requested 2024-11-14T14:00:43,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843444 2024-11-14T14:00:43,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843444 2024-11-14T14:00:43,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843444) roll requested 2024-11-14T14:00:43,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843452 2024-11-14T14:00:43,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843452 2024-11-14T14:00:43,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843452) roll requested 2024-11-14T14:00:43,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843460 2024-11-14T14:00:43,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843460 2024-11-14T14:00:43,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843460) roll requested 2024-11-14T14:00:43,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843467 2024-11-14T14:00:43,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843467 2024-11-14T14:00:43,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843467) roll requested 2024-11-14T14:00:43,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843475 2024-11-14T14:00:43,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843475 2024-11-14T14:00:43,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843475) roll requested 2024-11-14T14:00:43,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843483 2024-11-14T14:00:43,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843483 2024-11-14T14:00:43,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843483) roll requested 2024-11-14T14:00:43,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843490 2024-11-14T14:00:43,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843490 2024-11-14T14:00:43,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843490) roll requested 2024-11-14T14:00:43,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843497 2024-11-14T14:00:43,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843497 2024-11-14T14:00:43,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843497) roll requested 2024-11-14T14:00:43,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843504 2024-11-14T14:00:43,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843504 2024-11-14T14:00:43,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843504) roll requested 2024-11-14T14:00:43,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843512 2024-11-14T14:00:43,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843512 2024-11-14T14:00:43,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843512) roll requested 2024-11-14T14:00:43,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843518 2024-11-14T14:00:43,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843518 2024-11-14T14:00:43,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843518) roll requested 2024-11-14T14:00:43,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843526 2024-11-14T14:00:43,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843526 2024-11-14T14:00:43,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843526) roll requested 2024-11-14T14:00:43,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843533 2024-11-14T14:00:43,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843533 2024-11-14T14:00:43,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843533) roll requested 2024-11-14T14:00:43,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843541 2024-11-14T14:00:43,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843541 2024-11-14T14:00:43,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843541) roll requested 2024-11-14T14:00:43,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843548 2024-11-14T14:00:43,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843548 2024-11-14T14:00:43,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843548) roll requested 2024-11-14T14:00:43,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843554 2024-11-14T14:00:43,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843554 2024-11-14T14:00:43,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843554) roll requested 2024-11-14T14:00:43,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843562 2024-11-14T14:00:43,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843562 2024-11-14T14:00:43,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843562) roll requested 2024-11-14T14:00:43,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843569 2024-11-14T14:00:43,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843569 2024-11-14T14:00:43,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843569) roll requested 2024-11-14T14:00:43,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843577 2024-11-14T14:00:43,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843577 2024-11-14T14:00:43,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843577) roll requested 2024-11-14T14:00:43,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843585 2024-11-14T14:00:43,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843585 2024-11-14T14:00:43,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843585) roll requested 2024-11-14T14:00:43,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843591 2024-11-14T14:00:43,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843591 2024-11-14T14:00:43,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843591) roll requested 2024-11-14T14:00:43,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843599 2024-11-14T14:00:43,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843599 2024-11-14T14:00:43,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843599) roll requested 2024-11-14T14:00:43,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843606 2024-11-14T14:00:43,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843606 2024-11-14T14:00:43,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843606) roll requested 2024-11-14T14:00:43,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843613 2024-11-14T14:00:43,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843613 2024-11-14T14:00:43,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843613) roll requested 2024-11-14T14:00:43,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843620 2024-11-14T14:00:43,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843620 2024-11-14T14:00:43,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843620) roll requested 2024-11-14T14:00:43,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843628 2024-11-14T14:00:43,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843628 2024-11-14T14:00:43,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843628) roll requested 2024-11-14T14:00:43,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843636 2024-11-14T14:00:43,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843636 2024-11-14T14:00:43,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843636) roll requested 2024-11-14T14:00:43,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843643 2024-11-14T14:00:43,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843643 2024-11-14T14:00:43,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843643) roll requested 2024-11-14T14:00:43,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843649 2024-11-14T14:00:43,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843649 2024-11-14T14:00:43,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843649) roll requested 2024-11-14T14:00:43,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843656 2024-11-14T14:00:43,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843656 2024-11-14T14:00:43,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843656) roll requested 2024-11-14T14:00:43,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843664 2024-11-14T14:00:43,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843664 2024-11-14T14:00:43,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843664) roll requested 2024-11-14T14:00:43,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843671 2024-11-14T14:00:43,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843671 2024-11-14T14:00:43,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843671) roll requested 2024-11-14T14:00:43,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843678 2024-11-14T14:00:43,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843678 2024-11-14T14:00:43,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843678) roll requested 2024-11-14T14:00:43,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843686 2024-11-14T14:00:43,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843686 2024-11-14T14:00:43,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843686) roll requested 2024-11-14T14:00:43,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843693 2024-11-14T14:00:43,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843693 2024-11-14T14:00:43,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843693) roll requested 2024-11-14T14:00:43,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843704 2024-11-14T14:00:43,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843704 2024-11-14T14:00:43,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843704) roll requested 2024-11-14T14:00:43,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843713 2024-11-14T14:00:43,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843713 2024-11-14T14:00:43,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843713) roll requested 2024-11-14T14:00:43,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843720 2024-11-14T14:00:43,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843720 2024-11-14T14:00:43,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843720) roll requested 2024-11-14T14:00:43,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843728 2024-11-14T14:00:43,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843728 2024-11-14T14:00:43,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843728) roll requested 2024-11-14T14:00:43,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843734 2024-11-14T14:00:43,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843734 2024-11-14T14:00:43,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843734) roll requested 2024-11-14T14:00:43,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843741 2024-11-14T14:00:43,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843741 2024-11-14T14:00:43,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843741) roll requested 2024-11-14T14:00:43,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843749 2024-11-14T14:00:43,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843749 2024-11-14T14:00:43,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843749) roll requested 2024-11-14T14:00:43,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843758 2024-11-14T14:00:43,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843758 2024-11-14T14:00:43,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843758) roll requested 2024-11-14T14:00:43,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843766 2024-11-14T14:00:43,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843766 2024-11-14T14:00:43,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843766) roll requested 2024-11-14T14:00:43,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843774 2024-11-14T14:00:43,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843774 2024-11-14T14:00:43,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843774) roll requested 2024-11-14T14:00:43,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843781 2024-11-14T14:00:43,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843781 2024-11-14T14:00:43,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843781) roll requested 2024-11-14T14:00:43,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843788 2024-11-14T14:00:43,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843788 2024-11-14T14:00:43,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843788) roll requested 2024-11-14T14:00:43,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843796 2024-11-14T14:00:43,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843796 2024-11-14T14:00:43,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843796) roll requested 2024-11-14T14:00:43,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843803 2024-11-14T14:00:43,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843803 2024-11-14T14:00:43,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843803) roll requested 2024-11-14T14:00:43,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843810 2024-11-14T14:00:43,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843810 2024-11-14T14:00:43,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843810) roll requested 2024-11-14T14:00:43,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843818 2024-11-14T14:00:43,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843818 2024-11-14T14:00:43,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843818) roll requested 2024-11-14T14:00:43,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843825 2024-11-14T14:00:43,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843825 2024-11-14T14:00:43,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843825) roll requested 2024-11-14T14:00:43,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843833 2024-11-14T14:00:43,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843833 2024-11-14T14:00:43,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843833) roll requested 2024-11-14T14:00:43,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843841 2024-11-14T14:00:43,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843841 2024-11-14T14:00:43,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843841) roll requested 2024-11-14T14:00:43,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843848 2024-11-14T14:00:43,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843848 2024-11-14T14:00:43,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843848) roll requested 2024-11-14T14:00:43,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843856 2024-11-14T14:00:43,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843856 2024-11-14T14:00:43,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843856) roll requested 2024-11-14T14:00:43,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843869 2024-11-14T14:00:43,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843869 2024-11-14T14:00:43,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843869) roll requested 2024-11-14T14:00:43,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843878 2024-11-14T14:00:43,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843878 2024-11-14T14:00:43,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843878) roll requested 2024-11-14T14:00:43,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843891 2024-11-14T14:00:43,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843891 2024-11-14T14:00:43,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843891) roll requested 2024-11-14T14:00:43,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843899 2024-11-14T14:00:43,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843899 2024-11-14T14:00:43,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843899) roll requested 2024-11-14T14:00:43,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843912 2024-11-14T14:00:43,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843912 2024-11-14T14:00:43,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843912) roll requested 2024-11-14T14:00:43,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843919 2024-11-14T14:00:43,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843919 2024-11-14T14:00:43,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843919) roll requested 2024-11-14T14:00:43,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843929 2024-11-14T14:00:43,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843929 2024-11-14T14:00:43,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843929) roll requested 2024-11-14T14:00:43,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843937 2024-11-14T14:00:43,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843937 2024-11-14T14:00:43,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843937) roll requested 2024-11-14T14:00:43,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843949 2024-11-14T14:00:43,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843949 2024-11-14T14:00:43,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843949) roll requested 2024-11-14T14:00:43,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843958 2024-11-14T14:00:43,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843958 2024-11-14T14:00:43,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843958) roll requested 2024-11-14T14:00:43,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843965 2024-11-14T14:00:43,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843965 2024-11-14T14:00:43,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843965) roll requested 2024-11-14T14:00:43,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843975 2024-11-14T14:00:43,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843975 2024-11-14T14:00:43,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843975) roll requested 2024-11-14T14:00:43,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843982 2024-11-14T14:00:43,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843982 2024-11-14T14:00:43,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:43,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:43,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843982) roll requested 2024-11-14T14:00:43,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592843992 2024-11-14T14:00:43,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:43,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843992 2024-11-14T14:00:44,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592843992) roll requested 2024-11-14T14:00:44,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844000 2024-11-14T14:00:44,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592843992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844000 2024-11-14T14:00:44,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844000) roll requested 2024-11-14T14:00:44,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844010 2024-11-14T14:00:44,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844010 2024-11-14T14:00:44,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844010) roll requested 2024-11-14T14:00:44,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844021 2024-11-14T14:00:44,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844021 2024-11-14T14:00:44,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844021) roll requested 2024-11-14T14:00:44,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844033 2024-11-14T14:00:44,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844033 2024-11-14T14:00:44,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844033) roll requested 2024-11-14T14:00:44,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844041 2024-11-14T14:00:44,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844041 2024-11-14T14:00:44,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844041) roll requested 2024-11-14T14:00:44,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844049 2024-11-14T14:00:44,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844049 2024-11-14T14:00:44,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844049) roll requested 2024-11-14T14:00:44,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844056 2024-11-14T14:00:44,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844056 2024-11-14T14:00:44,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844056) roll requested 2024-11-14T14:00:44,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844063 2024-11-14T14:00:44,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844063 2024-11-14T14:00:44,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844063) roll requested 2024-11-14T14:00:44,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844071 2024-11-14T14:00:44,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844071 2024-11-14T14:00:44,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844071) roll requested 2024-11-14T14:00:44,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844085 2024-11-14T14:00:44,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844085 2024-11-14T14:00:44,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844085) roll requested 2024-11-14T14:00:44,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844093 2024-11-14T14:00:44,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844093 2024-11-14T14:00:44,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844093) roll requested 2024-11-14T14:00:44,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844105 2024-11-14T14:00:44,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844105 2024-11-14T14:00:44,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844105) roll requested 2024-11-14T14:00:44,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844113 2024-11-14T14:00:44,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844113 2024-11-14T14:00:44,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844113) roll requested 2024-11-14T14:00:44,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844121 2024-11-14T14:00:44,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844121 2024-11-14T14:00:44,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844121) roll requested 2024-11-14T14:00:44,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844129 2024-11-14T14:00:44,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844129 2024-11-14T14:00:44,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844129) roll requested 2024-11-14T14:00:44,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844136 2024-11-14T14:00:44,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844136 2024-11-14T14:00:44,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844136) roll requested 2024-11-14T14:00:44,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844144 2024-11-14T14:00:44,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844144 2024-11-14T14:00:44,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844144) roll requested 2024-11-14T14:00:44,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844152 2024-11-14T14:00:44,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844152 2024-11-14T14:00:44,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844152) roll requested 2024-11-14T14:00:44,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844160 2024-11-14T14:00:44,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844160 2024-11-14T14:00:44,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844160) roll requested 2024-11-14T14:00:44,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844169 2024-11-14T14:00:44,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844169 2024-11-14T14:00:44,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844169) roll requested 2024-11-14T14:00:44,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844177 2024-11-14T14:00:44,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844177 2024-11-14T14:00:44,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844177) roll requested 2024-11-14T14:00:44,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844184 2024-11-14T14:00:44,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844184 2024-11-14T14:00:44,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844184) roll requested 2024-11-14T14:00:44,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844192 2024-11-14T14:00:44,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844192 2024-11-14T14:00:44,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844192) roll requested 2024-11-14T14:00:44,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844200 2024-11-14T14:00:44,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844200 2024-11-14T14:00:44,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844200) roll requested 2024-11-14T14:00:44,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844208 2024-11-14T14:00:44,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844208 2024-11-14T14:00:44,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844208) roll requested 2024-11-14T14:00:44,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844216 2024-11-14T14:00:44,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844216 2024-11-14T14:00:44,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844216) roll requested 2024-11-14T14:00:44,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844223 2024-11-14T14:00:44,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844223 2024-11-14T14:00:44,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844223) roll requested 2024-11-14T14:00:44,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844229 2024-11-14T14:00:44,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844229 2024-11-14T14:00:44,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844229) roll requested 2024-11-14T14:00:44,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844236 2024-11-14T14:00:44,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844236 2024-11-14T14:00:44,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844236) roll requested 2024-11-14T14:00:44,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844244 2024-11-14T14:00:44,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844244 2024-11-14T14:00:44,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844244) roll requested 2024-11-14T14:00:44,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844251 2024-11-14T14:00:44,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844251 2024-11-14T14:00:44,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844251) roll requested 2024-11-14T14:00:44,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844259 2024-11-14T14:00:44,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844259 2024-11-14T14:00:44,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844259) roll requested 2024-11-14T14:00:44,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844267 2024-11-14T14:00:44,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844267 2024-11-14T14:00:44,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844267) roll requested 2024-11-14T14:00:44,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844275 2024-11-14T14:00:44,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844275 2024-11-14T14:00:44,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844275) roll requested 2024-11-14T14:00:44,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844282 2024-11-14T14:00:44,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844282 2024-11-14T14:00:44,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844282) roll requested 2024-11-14T14:00:44,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844290 2024-11-14T14:00:44,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844290 2024-11-14T14:00:44,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844290) roll requested 2024-11-14T14:00:44,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844297 2024-11-14T14:00:44,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844297 2024-11-14T14:00:44,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844297) roll requested 2024-11-14T14:00:44,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844305 2024-11-14T14:00:44,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844305 2024-11-14T14:00:44,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844305) roll requested 2024-11-14T14:00:44,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844313 2024-11-14T14:00:44,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844313 2024-11-14T14:00:44,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844313) roll requested 2024-11-14T14:00:44,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844320 2024-11-14T14:00:44,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844320 2024-11-14T14:00:44,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844320) roll requested 2024-11-14T14:00:44,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844328 2024-11-14T14:00:44,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844328 2024-11-14T14:00:44,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844328) roll requested 2024-11-14T14:00:44,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844335 2024-11-14T14:00:44,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844335 2024-11-14T14:00:44,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844335) roll requested 2024-11-14T14:00:44,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844341 2024-11-14T14:00:44,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844341 2024-11-14T14:00:44,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844341) roll requested 2024-11-14T14:00:44,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844349 2024-11-14T14:00:44,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844349 2024-11-14T14:00:44,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844349) roll requested 2024-11-14T14:00:44,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844356 2024-11-14T14:00:44,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844356 2024-11-14T14:00:44,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844356) roll requested 2024-11-14T14:00:44,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844363 2024-11-14T14:00:44,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844363 2024-11-14T14:00:44,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844363) roll requested 2024-11-14T14:00:44,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844371 2024-11-14T14:00:44,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844371 2024-11-14T14:00:44,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844371) roll requested 2024-11-14T14:00:44,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844378 2024-11-14T14:00:44,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844378 2024-11-14T14:00:44,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844378) roll requested 2024-11-14T14:00:44,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844384 2024-11-14T14:00:44,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844384 2024-11-14T14:00:44,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844384) roll requested 2024-11-14T14:00:44,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844390 2024-11-14T14:00:44,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844390 2024-11-14T14:00:44,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844390) roll requested 2024-11-14T14:00:44,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844396 2024-11-14T14:00:44,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844396 2024-11-14T14:00:44,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844396) roll requested 2024-11-14T14:00:44,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844402 2024-11-14T14:00:44,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844402 2024-11-14T14:00:44,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844402) roll requested 2024-11-14T14:00:44,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844411 2024-11-14T14:00:44,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844411 2024-11-14T14:00:44,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844411) roll requested 2024-11-14T14:00:44,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844418 2024-11-14T14:00:44,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844418 2024-11-14T14:00:44,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844418) roll requested 2024-11-14T14:00:44,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844428 2024-11-14T14:00:44,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844428 2024-11-14T14:00:44,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844428) roll requested 2024-11-14T14:00:44,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844435 2024-11-14T14:00:44,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844435 2024-11-14T14:00:44,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844435) roll requested 2024-11-14T14:00:44,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844441 2024-11-14T14:00:44,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844441 2024-11-14T14:00:44,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844441) roll requested 2024-11-14T14:00:44,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844450 2024-11-14T14:00:44,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844450 2024-11-14T14:00:44,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844450) roll requested 2024-11-14T14:00:44,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844456 2024-11-14T14:00:44,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844456 2024-11-14T14:00:44,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844456) roll requested 2024-11-14T14:00:44,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844462 2024-11-14T14:00:44,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844462 2024-11-14T14:00:44,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844462) roll requested 2024-11-14T14:00:44,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844470 2024-11-14T14:00:44,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844470 2024-11-14T14:00:44,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844470) roll requested 2024-11-14T14:00:44,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844476 2024-11-14T14:00:44,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844476 2024-11-14T14:00:44,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844476) roll requested 2024-11-14T14:00:44,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844483 2024-11-14T14:00:44,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844483 2024-11-14T14:00:44,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844483) roll requested 2024-11-14T14:00:44,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844490 2024-11-14T14:00:44,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844490 2024-11-14T14:00:44,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844490) roll requested 2024-11-14T14:00:44,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844496 2024-11-14T14:00:44,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844496 2024-11-14T14:00:44,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844496) roll requested 2024-11-14T14:00:44,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844502 2024-11-14T14:00:44,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844502 2024-11-14T14:00:44,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844502) roll requested 2024-11-14T14:00:44,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844508 2024-11-14T14:00:44,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844508 2024-11-14T14:00:44,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844508) roll requested 2024-11-14T14:00:44,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844515 2024-11-14T14:00:44,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844515 2024-11-14T14:00:44,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844515) roll requested 2024-11-14T14:00:44,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844521 2024-11-14T14:00:44,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844521 2024-11-14T14:00:44,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844521) roll requested 2024-11-14T14:00:44,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844527 2024-11-14T14:00:44,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844527 2024-11-14T14:00:44,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844527) roll requested 2024-11-14T14:00:44,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844533 2024-11-14T14:00:44,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844533 2024-11-14T14:00:44,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844533) roll requested 2024-11-14T14:00:44,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844539 2024-11-14T14:00:44,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844539 2024-11-14T14:00:44,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844539) roll requested 2024-11-14T14:00:44,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844546 2024-11-14T14:00:44,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844546 2024-11-14T14:00:44,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844546) roll requested 2024-11-14T14:00:44,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844553 2024-11-14T14:00:44,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844553 2024-11-14T14:00:44,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844553) roll requested 2024-11-14T14:00:44,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844560 2024-11-14T14:00:44,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844560 2024-11-14T14:00:44,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844560) roll requested 2024-11-14T14:00:44,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844566 2024-11-14T14:00:44,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844566 2024-11-14T14:00:44,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844566) roll requested 2024-11-14T14:00:44,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844572 2024-11-14T14:00:44,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844572 2024-11-14T14:00:44,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844572) roll requested 2024-11-14T14:00:44,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844581 2024-11-14T14:00:44,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844581 2024-11-14T14:00:44,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844581) roll requested 2024-11-14T14:00:44,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844589 2024-11-14T14:00:44,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844589 2024-11-14T14:00:44,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844589) roll requested 2024-11-14T14:00:44,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844595 2024-11-14T14:00:44,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844595 2024-11-14T14:00:44,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844595) roll requested 2024-11-14T14:00:44,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844602 2024-11-14T14:00:44,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844602 2024-11-14T14:00:44,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844602) roll requested 2024-11-14T14:00:44,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844612 2024-11-14T14:00:44,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844612 2024-11-14T14:00:44,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844612) roll requested 2024-11-14T14:00:44,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844618 2024-11-14T14:00:44,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844618 2024-11-14T14:00:44,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844618) roll requested 2024-11-14T14:00:44,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844626 2024-11-14T14:00:44,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844626 2024-11-14T14:00:44,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844626) roll requested 2024-11-14T14:00:44,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844632 2024-11-14T14:00:44,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844632 2024-11-14T14:00:44,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844632) roll requested 2024-11-14T14:00:44,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844639 2024-11-14T14:00:44,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844639 2024-11-14T14:00:44,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844639) roll requested 2024-11-14T14:00:44,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844645 2024-11-14T14:00:44,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844645 2024-11-14T14:00:44,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844645) roll requested 2024-11-14T14:00:44,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844652 2024-11-14T14:00:44,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844652 2024-11-14T14:00:44,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844652) roll requested 2024-11-14T14:00:44,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844658 2024-11-14T14:00:44,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844658 2024-11-14T14:00:44,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844658) roll requested 2024-11-14T14:00:44,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844664 2024-11-14T14:00:44,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844664 2024-11-14T14:00:44,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844664) roll requested 2024-11-14T14:00:44,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844670 2024-11-14T14:00:44,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844670 2024-11-14T14:00:44,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844670) roll requested 2024-11-14T14:00:44,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844677 2024-11-14T14:00:44,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844677 2024-11-14T14:00:44,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844677) roll requested 2024-11-14T14:00:44,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844683 2024-11-14T14:00:44,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844683 2024-11-14T14:00:44,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844683) roll requested 2024-11-14T14:00:44,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844689 2024-11-14T14:00:44,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844689 2024-11-14T14:00:44,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844689) roll requested 2024-11-14T14:00:44,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844695 2024-11-14T14:00:44,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844695 2024-11-14T14:00:44,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844695) roll requested 2024-11-14T14:00:44,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844701 2024-11-14T14:00:44,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844701 2024-11-14T14:00:44,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844701) roll requested 2024-11-14T14:00:44,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844707 2024-11-14T14:00:44,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844707 2024-11-14T14:00:44,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844707) roll requested 2024-11-14T14:00:44,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844714 2024-11-14T14:00:44,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844714 2024-11-14T14:00:44,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844714) roll requested 2024-11-14T14:00:44,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844720 2024-11-14T14:00:44,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844720 2024-11-14T14:00:44,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844720) roll requested 2024-11-14T14:00:44,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844727 2024-11-14T14:00:44,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844727 2024-11-14T14:00:44,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844727) roll requested 2024-11-14T14:00:44,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844733 2024-11-14T14:00:44,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844733 2024-11-14T14:00:44,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844733) roll requested 2024-11-14T14:00:44,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844739 2024-11-14T14:00:44,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844739 2024-11-14T14:00:44,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844739) roll requested 2024-11-14T14:00:44,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844746 2024-11-14T14:00:44,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844746 2024-11-14T14:00:44,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844746) roll requested 2024-11-14T14:00:44,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844752 2024-11-14T14:00:44,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844752 2024-11-14T14:00:44,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844752) roll requested 2024-11-14T14:00:44,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844758 2024-11-14T14:00:44,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844758 2024-11-14T14:00:44,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844758) roll requested 2024-11-14T14:00:44,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844765 2024-11-14T14:00:44,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844765 2024-11-14T14:00:44,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844765) roll requested 2024-11-14T14:00:44,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844771 2024-11-14T14:00:44,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844771 2024-11-14T14:00:44,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844771) roll requested 2024-11-14T14:00:44,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844777 2024-11-14T14:00:44,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844777 2024-11-14T14:00:44,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844777) roll requested 2024-11-14T14:00:44,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844783 2024-11-14T14:00:44,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844783 2024-11-14T14:00:44,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844783) roll requested 2024-11-14T14:00:44,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844789 2024-11-14T14:00:44,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844789 2024-11-14T14:00:44,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844789) roll requested 2024-11-14T14:00:44,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844795 2024-11-14T14:00:44,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844795 2024-11-14T14:00:44,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844795) roll requested 2024-11-14T14:00:44,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844801 2024-11-14T14:00:44,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844801 2024-11-14T14:00:44,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844801) roll requested 2024-11-14T14:00:44,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844807 2024-11-14T14:00:44,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844807 2024-11-14T14:00:44,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844807) roll requested 2024-11-14T14:00:44,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844814 2024-11-14T14:00:44,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844814 2024-11-14T14:00:44,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844814) roll requested 2024-11-14T14:00:44,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844820 2024-11-14T14:00:44,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844820 2024-11-14T14:00:44,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844820) roll requested 2024-11-14T14:00:44,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844826 2024-11-14T14:00:44,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844826 2024-11-14T14:00:44,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844826) roll requested 2024-11-14T14:00:44,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844832 2024-11-14T14:00:44,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844832 2024-11-14T14:00:44,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844832) roll requested 2024-11-14T14:00:44,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844844 2024-11-14T14:00:44,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844844 2024-11-14T14:00:44,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844844) roll requested 2024-11-14T14:00:44,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844851 2024-11-14T14:00:44,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844851 2024-11-14T14:00:44,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844851) roll requested 2024-11-14T14:00:44,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844863 2024-11-14T14:00:44,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844863 2024-11-14T14:00:44,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844863) roll requested 2024-11-14T14:00:44,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844870 2024-11-14T14:00:44,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844870 2024-11-14T14:00:44,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844870) roll requested 2024-11-14T14:00:44,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844881 2024-11-14T14:00:44,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844881 2024-11-14T14:00:44,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844881) roll requested 2024-11-14T14:00:44,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844887 2024-11-14T14:00:44,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844887 2024-11-14T14:00:44,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844887) roll requested 2024-11-14T14:00:44,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844894 2024-11-14T14:00:44,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844894 2024-11-14T14:00:44,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844894) roll requested 2024-11-14T14:00:44,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844900 2024-11-14T14:00:44,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844900 2024-11-14T14:00:44,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844900) roll requested 2024-11-14T14:00:44,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844907 2024-11-14T14:00:44,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844907 2024-11-14T14:00:44,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844907) roll requested 2024-11-14T14:00:44,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844913 2024-11-14T14:00:44,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844913 2024-11-14T14:00:44,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844913) roll requested 2024-11-14T14:00:44,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844919 2024-11-14T14:00:44,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844919 2024-11-14T14:00:44,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844919) roll requested 2024-11-14T14:00:44,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844925 2024-11-14T14:00:44,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844925 2024-11-14T14:00:44,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844925) roll requested 2024-11-14T14:00:44,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844931 2024-11-14T14:00:44,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844931 2024-11-14T14:00:44,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844931) roll requested 2024-11-14T14:00:44,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844937 2024-11-14T14:00:44,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844937 2024-11-14T14:00:44,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844937) roll requested 2024-11-14T14:00:44,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844943 2024-11-14T14:00:44,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844943 2024-11-14T14:00:44,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844943) roll requested 2024-11-14T14:00:44,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844950 2024-11-14T14:00:44,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844950 2024-11-14T14:00:44,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844950) roll requested 2024-11-14T14:00:44,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844956 2024-11-14T14:00:44,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844956 2024-11-14T14:00:44,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844956) roll requested 2024-11-14T14:00:44,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844962 2024-11-14T14:00:44,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844962 2024-11-14T14:00:44,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844962) roll requested 2024-11-14T14:00:44,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844968 2024-11-14T14:00:44,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844968 2024-11-14T14:00:44,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844968) roll requested 2024-11-14T14:00:44,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844974 2024-11-14T14:00:44,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844974 2024-11-14T14:00:44,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844974) roll requested 2024-11-14T14:00:44,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844981 2024-11-14T14:00:44,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844981 2024-11-14T14:00:44,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844981) roll requested 2024-11-14T14:00:44,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844987 2024-11-14T14:00:44,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844987 2024-11-14T14:00:44,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844987) roll requested 2024-11-14T14:00:44,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844993 2024-11-14T14:00:44,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:44,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844993 2024-11-14T14:00:44,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:44,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:44,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844993) roll requested 2024-11-14T14:00:44,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592844999 2024-11-14T14:00:45,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844999 2024-11-14T14:00:45,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592844999) roll requested 2024-11-14T14:00:45,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845005 2024-11-14T14:00:45,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592844999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845005 2024-11-14T14:00:45,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845005) roll requested 2024-11-14T14:00:45,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845011 2024-11-14T14:00:45,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845011 2024-11-14T14:00:45,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845011) roll requested 2024-11-14T14:00:45,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845018 2024-11-14T14:00:45,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845018 2024-11-14T14:00:45,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845018) roll requested 2024-11-14T14:00:45,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845024 2024-11-14T14:00:45,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845024 2024-11-14T14:00:45,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845024) roll requested 2024-11-14T14:00:45,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845030 2024-11-14T14:00:45,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845030 2024-11-14T14:00:45,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845030) roll requested 2024-11-14T14:00:45,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845036 2024-11-14T14:00:45,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845036 2024-11-14T14:00:45,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845036) roll requested 2024-11-14T14:00:45,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845043 2024-11-14T14:00:45,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845043 2024-11-14T14:00:45,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845043) roll requested 2024-11-14T14:00:45,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845051 2024-11-14T14:00:45,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845051 2024-11-14T14:00:45,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845051) roll requested 2024-11-14T14:00:45,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845057 2024-11-14T14:00:45,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845057 2024-11-14T14:00:45,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845057) roll requested 2024-11-14T14:00:45,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845063 2024-11-14T14:00:45,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845063 2024-11-14T14:00:45,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845063) roll requested 2024-11-14T14:00:45,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845069 2024-11-14T14:00:45,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845069 2024-11-14T14:00:45,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845069) roll requested 2024-11-14T14:00:45,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845075 2024-11-14T14:00:45,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845075 2024-11-14T14:00:45,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845075) roll requested 2024-11-14T14:00:45,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845081 2024-11-14T14:00:45,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845081 2024-11-14T14:00:45,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845081) roll requested 2024-11-14T14:00:45,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845087 2024-11-14T14:00:45,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845087 2024-11-14T14:00:45,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845087) roll requested 2024-11-14T14:00:45,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845093 2024-11-14T14:00:45,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845093 2024-11-14T14:00:45,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845093) roll requested 2024-11-14T14:00:45,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845100 2024-11-14T14:00:45,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845100 2024-11-14T14:00:45,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845100) roll requested 2024-11-14T14:00:45,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845106 2024-11-14T14:00:45,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845106 2024-11-14T14:00:45,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845106) roll requested 2024-11-14T14:00:45,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845112 2024-11-14T14:00:45,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845112 2024-11-14T14:00:45,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845112) roll requested 2024-11-14T14:00:45,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845118 2024-11-14T14:00:45,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845118 2024-11-14T14:00:45,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845118) roll requested 2024-11-14T14:00:45,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845124 2024-11-14T14:00:45,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845124 2024-11-14T14:00:45,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845124) roll requested 2024-11-14T14:00:45,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845130 2024-11-14T14:00:45,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845130 2024-11-14T14:00:45,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845130) roll requested 2024-11-14T14:00:45,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845137 2024-11-14T14:00:45,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845137 2024-11-14T14:00:45,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845137) roll requested 2024-11-14T14:00:45,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845143 2024-11-14T14:00:45,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845143 2024-11-14T14:00:45,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845143) roll requested 2024-11-14T14:00:45,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845150 2024-11-14T14:00:45,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845150 2024-11-14T14:00:45,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845150) roll requested 2024-11-14T14:00:45,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845155 2024-11-14T14:00:45,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845155 2024-11-14T14:00:45,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845155) roll requested 2024-11-14T14:00:45,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845162 2024-11-14T14:00:45,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845162 2024-11-14T14:00:45,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845162) roll requested 2024-11-14T14:00:45,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845168 2024-11-14T14:00:45,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845168 2024-11-14T14:00:45,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845168) roll requested 2024-11-14T14:00:45,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845174 2024-11-14T14:00:45,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845174 2024-11-14T14:00:45,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845174) roll requested 2024-11-14T14:00:45,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845180 2024-11-14T14:00:45,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845180 2024-11-14T14:00:45,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845180) roll requested 2024-11-14T14:00:45,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845186 2024-11-14T14:00:45,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845186 2024-11-14T14:00:45,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845186) roll requested 2024-11-14T14:00:45,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845193 2024-11-14T14:00:45,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845193 2024-11-14T14:00:45,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845193) roll requested 2024-11-14T14:00:45,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845199 2024-11-14T14:00:45,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845199 2024-11-14T14:00:45,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845199) roll requested 2024-11-14T14:00:45,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845205 2024-11-14T14:00:45,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845205 2024-11-14T14:00:45,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845205) roll requested 2024-11-14T14:00:45,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845211 2024-11-14T14:00:45,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845211 2024-11-14T14:00:45,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845211) roll requested 2024-11-14T14:00:45,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845217 2024-11-14T14:00:45,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845217 2024-11-14T14:00:45,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845217) roll requested 2024-11-14T14:00:45,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845223 2024-11-14T14:00:45,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845223 2024-11-14T14:00:45,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845223) roll requested 2024-11-14T14:00:45,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845229 2024-11-14T14:00:45,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845229 2024-11-14T14:00:45,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845229) roll requested 2024-11-14T14:00:45,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845235 2024-11-14T14:00:45,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845235 2024-11-14T14:00:45,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845235) roll requested 2024-11-14T14:00:45,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845241 2024-11-14T14:00:45,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845241 2024-11-14T14:00:45,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845241) roll requested 2024-11-14T14:00:45,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845249 2024-11-14T14:00:45,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845249 2024-11-14T14:00:45,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845249) roll requested 2024-11-14T14:00:45,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845255 2024-11-14T14:00:45,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845255 2024-11-14T14:00:45,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845255) roll requested 2024-11-14T14:00:45,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845261 2024-11-14T14:00:45,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845261 2024-11-14T14:00:45,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845261) roll requested 2024-11-14T14:00:45,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845267 2024-11-14T14:00:45,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845267 2024-11-14T14:00:45,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845267) roll requested 2024-11-14T14:00:45,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845273 2024-11-14T14:00:45,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845273 2024-11-14T14:00:45,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845273) roll requested 2024-11-14T14:00:45,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845279 2024-11-14T14:00:45,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845279 2024-11-14T14:00:45,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845279) roll requested 2024-11-14T14:00:45,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845285 2024-11-14T14:00:45,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845285 2024-11-14T14:00:45,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845285) roll requested 2024-11-14T14:00:45,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845292 2024-11-14T14:00:45,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845292 2024-11-14T14:00:45,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845292) roll requested 2024-11-14T14:00:45,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845297 2024-11-14T14:00:45,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845297 2024-11-14T14:00:45,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845297) roll requested 2024-11-14T14:00:45,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845304 2024-11-14T14:00:45,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845304 2024-11-14T14:00:45,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845304) roll requested 2024-11-14T14:00:45,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845310 2024-11-14T14:00:45,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845310 2024-11-14T14:00:45,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845310) roll requested 2024-11-14T14:00:45,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845316 2024-11-14T14:00:45,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845316 2024-11-14T14:00:45,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845316) roll requested 2024-11-14T14:00:45,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845322 2024-11-14T14:00:45,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845322 2024-11-14T14:00:45,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845322) roll requested 2024-11-14T14:00:45,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845328 2024-11-14T14:00:45,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845328 2024-11-14T14:00:45,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845328) roll requested 2024-11-14T14:00:45,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845334 2024-11-14T14:00:45,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845334 2024-11-14T14:00:45,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845334) roll requested 2024-11-14T14:00:45,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845341 2024-11-14T14:00:45,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845341 2024-11-14T14:00:45,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845341) roll requested 2024-11-14T14:00:45,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845347 2024-11-14T14:00:45,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845347 2024-11-14T14:00:45,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845347) roll requested 2024-11-14T14:00:45,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845353 2024-11-14T14:00:45,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845353 2024-11-14T14:00:45,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845353) roll requested 2024-11-14T14:00:45,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845359 2024-11-14T14:00:45,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845359 2024-11-14T14:00:45,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845359) roll requested 2024-11-14T14:00:45,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845365 2024-11-14T14:00:45,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845365 2024-11-14T14:00:45,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845365) roll requested 2024-11-14T14:00:45,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845372 2024-11-14T14:00:45,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845372 2024-11-14T14:00:45,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845372) roll requested 2024-11-14T14:00:45,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845378 2024-11-14T14:00:45,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845378 2024-11-14T14:00:45,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845378) roll requested 2024-11-14T14:00:45,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845385 2024-11-14T14:00:45,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845385 2024-11-14T14:00:45,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845385) roll requested 2024-11-14T14:00:45,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845392 2024-11-14T14:00:45,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845392 2024-11-14T14:00:45,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845392) roll requested 2024-11-14T14:00:45,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845399 2024-11-14T14:00:45,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845399 2024-11-14T14:00:45,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845399) roll requested 2024-11-14T14:00:45,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845405 2024-11-14T14:00:45,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845405 2024-11-14T14:00:45,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845405) roll requested 2024-11-14T14:00:45,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845411 2024-11-14T14:00:45,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845411 2024-11-14T14:00:45,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845411) roll requested 2024-11-14T14:00:45,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845417 2024-11-14T14:00:45,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845417 2024-11-14T14:00:45,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845417) roll requested 2024-11-14T14:00:45,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845424 2024-11-14T14:00:45,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845424 2024-11-14T14:00:45,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845424) roll requested 2024-11-14T14:00:45,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845431 2024-11-14T14:00:45,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845431 2024-11-14T14:00:45,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845431) roll requested 2024-11-14T14:00:45,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845437 2024-11-14T14:00:45,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845437 2024-11-14T14:00:45,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845437) roll requested 2024-11-14T14:00:45,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845443 2024-11-14T14:00:45,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845443 2024-11-14T14:00:45,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845443) roll requested 2024-11-14T14:00:45,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845450 2024-11-14T14:00:45,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845450 2024-11-14T14:00:45,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845450) roll requested 2024-11-14T14:00:45,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845456 2024-11-14T14:00:45,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845456 2024-11-14T14:00:45,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845456) roll requested 2024-11-14T14:00:45,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845463 2024-11-14T14:00:45,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845463 2024-11-14T14:00:45,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845463) roll requested 2024-11-14T14:00:45,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845469 2024-11-14T14:00:45,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845469 2024-11-14T14:00:45,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845469) roll requested 2024-11-14T14:00:45,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845475 2024-11-14T14:00:45,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845475 2024-11-14T14:00:45,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845475) roll requested 2024-11-14T14:00:45,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845481 2024-11-14T14:00:45,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845481 2024-11-14T14:00:45,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845481) roll requested 2024-11-14T14:00:45,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845487 2024-11-14T14:00:45,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845487 2024-11-14T14:00:45,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845487) roll requested 2024-11-14T14:00:45,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845493 2024-11-14T14:00:45,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845493 2024-11-14T14:00:45,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845493) roll requested 2024-11-14T14:00:45,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845500 2024-11-14T14:00:45,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845500 2024-11-14T14:00:45,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845500) roll requested 2024-11-14T14:00:45,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845506 2024-11-14T14:00:45,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845506 2024-11-14T14:00:45,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845506) roll requested 2024-11-14T14:00:45,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845513 2024-11-14T14:00:45,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845513 2024-11-14T14:00:45,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845513) roll requested 2024-11-14T14:00:45,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845519 2024-11-14T14:00:45,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845519 2024-11-14T14:00:45,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845519) roll requested 2024-11-14T14:00:45,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845525 2024-11-14T14:00:45,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845525 2024-11-14T14:00:45,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845525) roll requested 2024-11-14T14:00:45,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845531 2024-11-14T14:00:45,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845531 2024-11-14T14:00:45,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845531) roll requested 2024-11-14T14:00:45,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845537 2024-11-14T14:00:45,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845537 2024-11-14T14:00:45,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845537) roll requested 2024-11-14T14:00:45,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845544 2024-11-14T14:00:45,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845544 2024-11-14T14:00:45,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845544) roll requested 2024-11-14T14:00:45,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845550 2024-11-14T14:00:45,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845550 2024-11-14T14:00:45,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845550) roll requested 2024-11-14T14:00:45,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845556 2024-11-14T14:00:45,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845556 2024-11-14T14:00:45,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845556) roll requested 2024-11-14T14:00:45,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845562 2024-11-14T14:00:45,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845562 2024-11-14T14:00:45,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845562) roll requested 2024-11-14T14:00:45,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845568 2024-11-14T14:00:45,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845568 2024-11-14T14:00:45,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845568) roll requested 2024-11-14T14:00:45,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845574 2024-11-14T14:00:45,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845574 2024-11-14T14:00:45,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845574) roll requested 2024-11-14T14:00:45,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845581 2024-11-14T14:00:45,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845581 2024-11-14T14:00:45,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845581) roll requested 2024-11-14T14:00:45,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845587 2024-11-14T14:00:45,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845587 2024-11-14T14:00:45,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845587) roll requested 2024-11-14T14:00:45,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845593 2024-11-14T14:00:45,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845593 2024-11-14T14:00:45,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845593) roll requested 2024-11-14T14:00:45,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845601 2024-11-14T14:00:45,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845601 2024-11-14T14:00:45,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845601) roll requested 2024-11-14T14:00:45,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845608 2024-11-14T14:00:45,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845608 2024-11-14T14:00:45,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845608) roll requested 2024-11-14T14:00:45,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845614 2024-11-14T14:00:45,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845614 2024-11-14T14:00:45,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845614) roll requested 2024-11-14T14:00:45,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845620 2024-11-14T14:00:45,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845620 2024-11-14T14:00:45,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845620) roll requested 2024-11-14T14:00:45,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845627 2024-11-14T14:00:45,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845627 2024-11-14T14:00:45,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845627) roll requested 2024-11-14T14:00:45,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845632 2024-11-14T14:00:45,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845632 2024-11-14T14:00:45,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845632) roll requested 2024-11-14T14:00:45,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845638 2024-11-14T14:00:45,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845638 2024-11-14T14:00:45,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845638) roll requested 2024-11-14T14:00:45,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845645 2024-11-14T14:00:45,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845645 2024-11-14T14:00:45,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845645) roll requested 2024-11-14T14:00:45,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845651 2024-11-14T14:00:45,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845651 2024-11-14T14:00:45,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845651) roll requested 2024-11-14T14:00:45,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845657 2024-11-14T14:00:45,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845657 2024-11-14T14:00:45,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845657) roll requested 2024-11-14T14:00:45,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845663 2024-11-14T14:00:45,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845663 2024-11-14T14:00:45,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845663) roll requested 2024-11-14T14:00:45,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845670 2024-11-14T14:00:45,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845670 2024-11-14T14:00:45,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845670) roll requested 2024-11-14T14:00:45,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845676 2024-11-14T14:00:45,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845676 2024-11-14T14:00:45,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845676) roll requested 2024-11-14T14:00:45,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845682 2024-11-14T14:00:45,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845682 2024-11-14T14:00:45,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845682) roll requested 2024-11-14T14:00:45,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845688 2024-11-14T14:00:45,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845688 2024-11-14T14:00:45,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845688) roll requested 2024-11-14T14:00:45,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845695 2024-11-14T14:00:45,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845695 2024-11-14T14:00:45,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845695) roll requested 2024-11-14T14:00:45,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845702 2024-11-14T14:00:45,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845702 2024-11-14T14:00:45,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845702) roll requested 2024-11-14T14:00:45,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845708 2024-11-14T14:00:45,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845708 2024-11-14T14:00:45,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845708) roll requested 2024-11-14T14:00:45,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845714 2024-11-14T14:00:45,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845714 2024-11-14T14:00:45,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845714) roll requested 2024-11-14T14:00:45,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845721 2024-11-14T14:00:45,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845721 2024-11-14T14:00:45,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845721) roll requested 2024-11-14T14:00:45,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845727 2024-11-14T14:00:45,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845727 2024-11-14T14:00:45,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845727) roll requested 2024-11-14T14:00:45,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845733 2024-11-14T14:00:45,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845733 2024-11-14T14:00:45,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845733) roll requested 2024-11-14T14:00:45,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845739 2024-11-14T14:00:45,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845739 2024-11-14T14:00:45,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845739) roll requested 2024-11-14T14:00:45,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845745 2024-11-14T14:00:45,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845745 2024-11-14T14:00:45,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845745) roll requested 2024-11-14T14:00:45,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845751 2024-11-14T14:00:45,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845751 2024-11-14T14:00:45,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845751) roll requested 2024-11-14T14:00:45,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845757 2024-11-14T14:00:45,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845757 2024-11-14T14:00:45,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845757) roll requested 2024-11-14T14:00:45,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845764 2024-11-14T14:00:45,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845764 2024-11-14T14:00:45,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845764) roll requested 2024-11-14T14:00:45,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845770 2024-11-14T14:00:45,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845770 2024-11-14T14:00:45,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845770) roll requested 2024-11-14T14:00:45,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845776 2024-11-14T14:00:45,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845776 2024-11-14T14:00:45,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845776) roll requested 2024-11-14T14:00:45,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845782 2024-11-14T14:00:45,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845782 2024-11-14T14:00:45,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845782) roll requested 2024-11-14T14:00:45,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845788 2024-11-14T14:00:45,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845788 2024-11-14T14:00:45,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845788) roll requested 2024-11-14T14:00:45,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845794 2024-11-14T14:00:45,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845794 2024-11-14T14:00:45,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845794) roll requested 2024-11-14T14:00:45,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845800 2024-11-14T14:00:45,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845800 2024-11-14T14:00:45,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845800) roll requested 2024-11-14T14:00:45,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845806 2024-11-14T14:00:45,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845806 2024-11-14T14:00:45,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845806) roll requested 2024-11-14T14:00:45,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845813 2024-11-14T14:00:45,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845813 2024-11-14T14:00:45,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845813) roll requested 2024-11-14T14:00:45,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845819 2024-11-14T14:00:45,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845819 2024-11-14T14:00:45,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845819) roll requested 2024-11-14T14:00:45,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845826 2024-11-14T14:00:45,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845826 2024-11-14T14:00:45,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845826) roll requested 2024-11-14T14:00:45,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845832 2024-11-14T14:00:45,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845832 2024-11-14T14:00:45,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845832) roll requested 2024-11-14T14:00:45,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845838 2024-11-14T14:00:45,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845838 2024-11-14T14:00:45,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845838) roll requested 2024-11-14T14:00:45,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845844 2024-11-14T14:00:45,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845844 2024-11-14T14:00:45,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845844) roll requested 2024-11-14T14:00:45,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845851 2024-11-14T14:00:45,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845851 2024-11-14T14:00:45,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845851) roll requested 2024-11-14T14:00:45,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845857 2024-11-14T14:00:45,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845857 2024-11-14T14:00:45,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845857) roll requested 2024-11-14T14:00:45,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845863 2024-11-14T14:00:45,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845863 2024-11-14T14:00:45,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845863) roll requested 2024-11-14T14:00:45,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845869 2024-11-14T14:00:45,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845869 2024-11-14T14:00:45,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845869) roll requested 2024-11-14T14:00:45,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845880 2024-11-14T14:00:45,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845880 2024-11-14T14:00:45,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845880) roll requested 2024-11-14T14:00:45,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845886 2024-11-14T14:00:45,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845886 2024-11-14T14:00:45,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845886) roll requested 2024-11-14T14:00:45,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845892 2024-11-14T14:00:45,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845892 2024-11-14T14:00:45,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845892) roll requested 2024-11-14T14:00:45,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845898 2024-11-14T14:00:45,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845898 2024-11-14T14:00:45,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845898) roll requested 2024-11-14T14:00:45,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845905 2024-11-14T14:00:45,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845905 2024-11-14T14:00:45,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845905) roll requested 2024-11-14T14:00:45,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845911 2024-11-14T14:00:45,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845911 2024-11-14T14:00:45,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845911) roll requested 2024-11-14T14:00:45,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845917 2024-11-14T14:00:45,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845917 2024-11-14T14:00:45,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845917) roll requested 2024-11-14T14:00:45,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845923 2024-11-14T14:00:45,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845923 2024-11-14T14:00:45,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845923) roll requested 2024-11-14T14:00:45,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845929 2024-11-14T14:00:45,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845929 2024-11-14T14:00:45,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845929) roll requested 2024-11-14T14:00:45,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845935 2024-11-14T14:00:45,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845935 2024-11-14T14:00:45,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845935) roll requested 2024-11-14T14:00:45,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845941 2024-11-14T14:00:45,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845941 2024-11-14T14:00:45,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845941) roll requested 2024-11-14T14:00:45,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845947 2024-11-14T14:00:45,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845947 2024-11-14T14:00:45,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845947) roll requested 2024-11-14T14:00:45,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845954 2024-11-14T14:00:45,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845954 2024-11-14T14:00:45,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845954) roll requested 2024-11-14T14:00:45,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845960 2024-11-14T14:00:45,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845960 2024-11-14T14:00:45,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845960) roll requested 2024-11-14T14:00:45,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845966 2024-11-14T14:00:45,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845966 2024-11-14T14:00:45,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845966) roll requested 2024-11-14T14:00:45,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845972 2024-11-14T14:00:45,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845972 2024-11-14T14:00:45,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845972) roll requested 2024-11-14T14:00:45,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845978 2024-11-14T14:00:45,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845978 2024-11-14T14:00:45,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845978) roll requested 2024-11-14T14:00:45,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845984 2024-11-14T14:00:45,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845984 2024-11-14T14:00:45,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845984) roll requested 2024-11-14T14:00:45,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845990 2024-11-14T14:00:45,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:45,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845990 2024-11-14T14:00:45,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:45,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:45,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845990) roll requested 2024-11-14T14:00:45,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592845996 2024-11-14T14:00:46,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845996 2024-11-14T14:00:46,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592845996) roll requested 2024-11-14T14:00:46,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846002 2024-11-14T14:00:46,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592845996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846002 2024-11-14T14:00:46,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846002) roll requested 2024-11-14T14:00:46,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846008 2024-11-14T14:00:46,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846008 2024-11-14T14:00:46,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846008) roll requested 2024-11-14T14:00:46,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846014 2024-11-14T14:00:46,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846014 2024-11-14T14:00:46,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846014) roll requested 2024-11-14T14:00:46,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846021 2024-11-14T14:00:46,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846021 2024-11-14T14:00:46,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846021) roll requested 2024-11-14T14:00:46,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846026 2024-11-14T14:00:46,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846026 2024-11-14T14:00:46,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846026) roll requested 2024-11-14T14:00:46,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846033 2024-11-14T14:00:46,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846033 2024-11-14T14:00:46,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846033) roll requested 2024-11-14T14:00:46,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846039 2024-11-14T14:00:46,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846039 2024-11-14T14:00:46,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846039) roll requested 2024-11-14T14:00:46,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846047 2024-11-14T14:00:46,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846047 2024-11-14T14:00:46,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846047) roll requested 2024-11-14T14:00:46,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846053 2024-11-14T14:00:46,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846053 2024-11-14T14:00:46,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846053) roll requested 2024-11-14T14:00:46,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846059 2024-11-14T14:00:46,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846059 2024-11-14T14:00:46,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846059) roll requested 2024-11-14T14:00:46,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846065 2024-11-14T14:00:46,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846065 2024-11-14T14:00:46,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846065) roll requested 2024-11-14T14:00:46,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846071 2024-11-14T14:00:46,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846071 2024-11-14T14:00:46,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846071) roll requested 2024-11-14T14:00:46,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846077 2024-11-14T14:00:46,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846077 2024-11-14T14:00:46,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846077) roll requested 2024-11-14T14:00:46,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846083 2024-11-14T14:00:46,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846083 2024-11-14T14:00:46,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846083) roll requested 2024-11-14T14:00:46,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846089 2024-11-14T14:00:46,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846089 2024-11-14T14:00:46,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846089) roll requested 2024-11-14T14:00:46,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846095 2024-11-14T14:00:46,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846095 2024-11-14T14:00:46,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846095) roll requested 2024-11-14T14:00:46,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846102 2024-11-14T14:00:46,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846102 2024-11-14T14:00:46,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846102) roll requested 2024-11-14T14:00:46,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846108 2024-11-14T14:00:46,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846108 2024-11-14T14:00:46,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846108) roll requested 2024-11-14T14:00:46,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846114 2024-11-14T14:00:46,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846114 2024-11-14T14:00:46,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846114) roll requested 2024-11-14T14:00:46,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846120 2024-11-14T14:00:46,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846120 2024-11-14T14:00:46,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846120) roll requested 2024-11-14T14:00:46,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846126 2024-11-14T14:00:46,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846126 2024-11-14T14:00:46,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846126) roll requested 2024-11-14T14:00:46,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846133 2024-11-14T14:00:46,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846133 2024-11-14T14:00:46,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846133) roll requested 2024-11-14T14:00:46,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846139 2024-11-14T14:00:46,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846139 2024-11-14T14:00:46,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846139) roll requested 2024-11-14T14:00:46,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846145 2024-11-14T14:00:46,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846145 2024-11-14T14:00:46,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846145) roll requested 2024-11-14T14:00:46,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846151 2024-11-14T14:00:46,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846151 2024-11-14T14:00:46,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846151) roll requested 2024-11-14T14:00:46,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846157 2024-11-14T14:00:46,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846157 2024-11-14T14:00:46,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846157) roll requested 2024-11-14T14:00:46,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846165 2024-11-14T14:00:46,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846165 2024-11-14T14:00:46,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846165) roll requested 2024-11-14T14:00:46,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846171 2024-11-14T14:00:46,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846171 2024-11-14T14:00:46,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846171) roll requested 2024-11-14T14:00:46,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846178 2024-11-14T14:00:46,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846178 2024-11-14T14:00:46,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846178) roll requested 2024-11-14T14:00:46,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846185 2024-11-14T14:00:46,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846185 2024-11-14T14:00:46,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846185) roll requested 2024-11-14T14:00:46,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846191 2024-11-14T14:00:46,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846191 2024-11-14T14:00:46,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846191) roll requested 2024-11-14T14:00:46,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846197 2024-11-14T14:00:46,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846197 2024-11-14T14:00:46,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846197) roll requested 2024-11-14T14:00:46,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846203 2024-11-14T14:00:46,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846203 2024-11-14T14:00:46,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846203) roll requested 2024-11-14T14:00:46,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846209 2024-11-14T14:00:46,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846209 2024-11-14T14:00:46,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846209) roll requested 2024-11-14T14:00:46,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846216 2024-11-14T14:00:46,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846216 2024-11-14T14:00:46,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846216) roll requested 2024-11-14T14:00:46,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846222 2024-11-14T14:00:46,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846222 2024-11-14T14:00:46,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846222) roll requested 2024-11-14T14:00:46,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846228 2024-11-14T14:00:46,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846228 2024-11-14T14:00:46,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846228) roll requested 2024-11-14T14:00:46,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846234 2024-11-14T14:00:46,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846234 2024-11-14T14:00:46,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846234) roll requested 2024-11-14T14:00:46,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846240 2024-11-14T14:00:46,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846240 2024-11-14T14:00:46,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846240) roll requested 2024-11-14T14:00:46,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846246 2024-11-14T14:00:46,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846246 2024-11-14T14:00:46,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846246) roll requested 2024-11-14T14:00:46,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846253 2024-11-14T14:00:46,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846253 2024-11-14T14:00:46,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846253) roll requested 2024-11-14T14:00:46,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846259 2024-11-14T14:00:46,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846259 2024-11-14T14:00:46,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846259) roll requested 2024-11-14T14:00:46,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846265 2024-11-14T14:00:46,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846265 2024-11-14T14:00:46,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846265) roll requested 2024-11-14T14:00:46,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846271 2024-11-14T14:00:46,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846271 2024-11-14T14:00:46,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846271) roll requested 2024-11-14T14:00:46,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846277 2024-11-14T14:00:46,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846277 2024-11-14T14:00:46,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846277) roll requested 2024-11-14T14:00:46,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846283 2024-11-14T14:00:46,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846283 2024-11-14T14:00:46,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846283) roll requested 2024-11-14T14:00:46,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846289 2024-11-14T14:00:46,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846289 2024-11-14T14:00:46,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846289) roll requested 2024-11-14T14:00:46,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846295 2024-11-14T14:00:46,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846295 2024-11-14T14:00:46,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846295) roll requested 2024-11-14T14:00:46,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846302 2024-11-14T14:00:46,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846302 2024-11-14T14:00:46,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846302) roll requested 2024-11-14T14:00:46,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846308 2024-11-14T14:00:46,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846308 2024-11-14T14:00:46,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846308) roll requested 2024-11-14T14:00:46,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846315 2024-11-14T14:00:46,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846315 2024-11-14T14:00:46,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846315) roll requested 2024-11-14T14:00:46,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846321 2024-11-14T14:00:46,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846321 2024-11-14T14:00:46,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846321) roll requested 2024-11-14T14:00:46,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846327 2024-11-14T14:00:46,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846327 2024-11-14T14:00:46,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846327) roll requested 2024-11-14T14:00:46,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846333 2024-11-14T14:00:46,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846333 2024-11-14T14:00:46,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846333) roll requested 2024-11-14T14:00:46,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846339 2024-11-14T14:00:46,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846339 2024-11-14T14:00:46,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846339) roll requested 2024-11-14T14:00:46,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846345 2024-11-14T14:00:46,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846345 2024-11-14T14:00:46,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846345) roll requested 2024-11-14T14:00:46,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846352 2024-11-14T14:00:46,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846352 2024-11-14T14:00:46,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846352) roll requested 2024-11-14T14:00:46,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846358 2024-11-14T14:00:46,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846358 2024-11-14T14:00:46,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846358) roll requested 2024-11-14T14:00:46,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846364 2024-11-14T14:00:46,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846364 2024-11-14T14:00:46,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846364) roll requested 2024-11-14T14:00:46,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846370 2024-11-14T14:00:46,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846370 2024-11-14T14:00:46,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846370) roll requested 2024-11-14T14:00:46,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846376 2024-11-14T14:00:46,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846376 2024-11-14T14:00:46,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846376) roll requested 2024-11-14T14:00:46,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846382 2024-11-14T14:00:46,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846382 2024-11-14T14:00:46,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846382) roll requested 2024-11-14T14:00:46,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846389 2024-11-14T14:00:46,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846389 2024-11-14T14:00:46,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846389) roll requested 2024-11-14T14:00:46,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846395 2024-11-14T14:00:46,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846395 2024-11-14T14:00:46,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846395) roll requested 2024-11-14T14:00:46,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846401 2024-11-14T14:00:46,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846401 2024-11-14T14:00:46,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846401) roll requested 2024-11-14T14:00:46,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846407 2024-11-14T14:00:46,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846407 2024-11-14T14:00:46,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846407) roll requested 2024-11-14T14:00:46,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846413 2024-11-14T14:00:46,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846413 2024-11-14T14:00:46,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846413) roll requested 2024-11-14T14:00:46,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846419 2024-11-14T14:00:46,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846419 2024-11-14T14:00:46,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846419) roll requested 2024-11-14T14:00:46,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846426 2024-11-14T14:00:46,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846426 2024-11-14T14:00:46,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846426) roll requested 2024-11-14T14:00:46,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846432 2024-11-14T14:00:46,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846432 2024-11-14T14:00:46,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846432) roll requested 2024-11-14T14:00:46,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846438 2024-11-14T14:00:46,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846438 2024-11-14T14:00:46,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846438) roll requested 2024-11-14T14:00:46,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846444 2024-11-14T14:00:46,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846444 2024-11-14T14:00:46,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846444) roll requested 2024-11-14T14:00:46,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846450 2024-11-14T14:00:46,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846450 2024-11-14T14:00:46,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846450) roll requested 2024-11-14T14:00:46,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846456 2024-11-14T14:00:46,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846456 2024-11-14T14:00:46,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846456) roll requested 2024-11-14T14:00:46,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846462 2024-11-14T14:00:46,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846462 2024-11-14T14:00:46,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846462) roll requested 2024-11-14T14:00:46,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846468 2024-11-14T14:00:46,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846468 2024-11-14T14:00:46,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846468) roll requested 2024-11-14T14:00:46,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846474 2024-11-14T14:00:46,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846474 2024-11-14T14:00:46,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846474) roll requested 2024-11-14T14:00:46,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846480 2024-11-14T14:00:46,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846480 2024-11-14T14:00:46,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846480) roll requested 2024-11-14T14:00:46,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846487 2024-11-14T14:00:46,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846487 2024-11-14T14:00:46,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846487) roll requested 2024-11-14T14:00:46,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846493 2024-11-14T14:00:46,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846493 2024-11-14T14:00:46,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846493) roll requested 2024-11-14T14:00:46,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846499 2024-11-14T14:00:46,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846499 2024-11-14T14:00:46,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846499) roll requested 2024-11-14T14:00:46,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846505 2024-11-14T14:00:46,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846505 2024-11-14T14:00:46,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846505) roll requested 2024-11-14T14:00:46,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846511 2024-11-14T14:00:46,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846511 2024-11-14T14:00:46,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846511) roll requested 2024-11-14T14:00:46,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846517 2024-11-14T14:00:46,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846517 2024-11-14T14:00:46,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846517) roll requested 2024-11-14T14:00:46,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846523 2024-11-14T14:00:46,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846523 2024-11-14T14:00:46,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846523) roll requested 2024-11-14T14:00:46,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846530 2024-11-14T14:00:46,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846530 2024-11-14T14:00:46,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846530) roll requested 2024-11-14T14:00:46,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846536 2024-11-14T14:00:46,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846536 2024-11-14T14:00:46,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846536) roll requested 2024-11-14T14:00:46,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846542 2024-11-14T14:00:46,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846542 2024-11-14T14:00:46,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846542) roll requested 2024-11-14T14:00:46,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846548 2024-11-14T14:00:46,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846548 2024-11-14T14:00:46,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846548) roll requested 2024-11-14T14:00:46,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846555 2024-11-14T14:00:46,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846555 2024-11-14T14:00:46,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846555) roll requested 2024-11-14T14:00:46,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846561 2024-11-14T14:00:46,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846561 2024-11-14T14:00:46,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846561) roll requested 2024-11-14T14:00:46,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846568 2024-11-14T14:00:46,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846568 2024-11-14T14:00:46,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846568) roll requested 2024-11-14T14:00:46,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846574 2024-11-14T14:00:46,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846574 2024-11-14T14:00:46,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846574) roll requested 2024-11-14T14:00:46,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846580 2024-11-14T14:00:46,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846580 2024-11-14T14:00:46,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846580) roll requested 2024-11-14T14:00:46,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846586 2024-11-14T14:00:46,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846586 2024-11-14T14:00:46,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846586) roll requested 2024-11-14T14:00:46,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846592 2024-11-14T14:00:46,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846592 2024-11-14T14:00:46,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846592) roll requested 2024-11-14T14:00:46,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846598 2024-11-14T14:00:46,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846598 2024-11-14T14:00:46,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846598) roll requested 2024-11-14T14:00:46,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846604 2024-11-14T14:00:46,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846604 2024-11-14T14:00:46,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846604) roll requested 2024-11-14T14:00:46,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846610 2024-11-14T14:00:46,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846610 2024-11-14T14:00:46,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846610) roll requested 2024-11-14T14:00:46,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846617 2024-11-14T14:00:46,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846617 2024-11-14T14:00:46,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846617) roll requested 2024-11-14T14:00:46,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846623 2024-11-14T14:00:46,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846623 2024-11-14T14:00:46,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846623) roll requested 2024-11-14T14:00:46,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846629 2024-11-14T14:00:46,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846629 2024-11-14T14:00:46,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846629) roll requested 2024-11-14T14:00:46,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846635 2024-11-14T14:00:46,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846635 2024-11-14T14:00:46,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846635) roll requested 2024-11-14T14:00:46,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846641 2024-11-14T14:00:46,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846641 2024-11-14T14:00:46,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846641) roll requested 2024-11-14T14:00:46,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846647 2024-11-14T14:00:46,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846647 2024-11-14T14:00:46,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846647) roll requested 2024-11-14T14:00:46,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846653 2024-11-14T14:00:46,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846653 2024-11-14T14:00:46,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846653) roll requested 2024-11-14T14:00:46,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846659 2024-11-14T14:00:46,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846659 2024-11-14T14:00:46,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846659) roll requested 2024-11-14T14:00:46,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846665 2024-11-14T14:00:46,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846665 2024-11-14T14:00:46,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846665) roll requested 2024-11-14T14:00:46,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846672 2024-11-14T14:00:46,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846672 2024-11-14T14:00:46,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846672) roll requested 2024-11-14T14:00:46,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846678 2024-11-14T14:00:46,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846678 2024-11-14T14:00:46,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846678) roll requested 2024-11-14T14:00:46,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846684 2024-11-14T14:00:46,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846684 2024-11-14T14:00:46,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846684) roll requested 2024-11-14T14:00:46,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846690 2024-11-14T14:00:46,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846690 2024-11-14T14:00:46,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846690) roll requested 2024-11-14T14:00:46,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846696 2024-11-14T14:00:46,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846696 2024-11-14T14:00:46,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846696) roll requested 2024-11-14T14:00:46,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846704 2024-11-14T14:00:46,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846704 2024-11-14T14:00:46,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846704) roll requested 2024-11-14T14:00:46,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846711 2024-11-14T14:00:46,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846711 2024-11-14T14:00:46,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846711) roll requested 2024-11-14T14:00:46,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846717 2024-11-14T14:00:46,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846717 2024-11-14T14:00:46,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846717) roll requested 2024-11-14T14:00:46,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846723 2024-11-14T14:00:46,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846723 2024-11-14T14:00:46,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846723) roll requested 2024-11-14T14:00:46,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846730 2024-11-14T14:00:46,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846730 2024-11-14T14:00:46,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846730) roll requested 2024-11-14T14:00:46,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846736 2024-11-14T14:00:46,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846736 2024-11-14T14:00:46,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846736) roll requested 2024-11-14T14:00:46,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846742 2024-11-14T14:00:46,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846742 2024-11-14T14:00:46,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846742) roll requested 2024-11-14T14:00:46,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846748 2024-11-14T14:00:46,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846748 2024-11-14T14:00:46,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846748) roll requested 2024-11-14T14:00:46,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846754 2024-11-14T14:00:46,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846754 2024-11-14T14:00:46,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846754) roll requested 2024-11-14T14:00:46,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846761 2024-11-14T14:00:46,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846761 2024-11-14T14:00:46,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846761) roll requested 2024-11-14T14:00:46,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846767 2024-11-14T14:00:46,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846767 2024-11-14T14:00:46,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846767) roll requested 2024-11-14T14:00:46,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846773 2024-11-14T14:00:46,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846773 2024-11-14T14:00:46,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846773) roll requested 2024-11-14T14:00:46,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846779 2024-11-14T14:00:46,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846779 2024-11-14T14:00:46,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846779) roll requested 2024-11-14T14:00:46,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846785 2024-11-14T14:00:46,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846785 2024-11-14T14:00:46,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846785) roll requested 2024-11-14T14:00:46,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846791 2024-11-14T14:00:46,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846791 2024-11-14T14:00:46,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846791) roll requested 2024-11-14T14:00:46,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846798 2024-11-14T14:00:46,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846798 2024-11-14T14:00:46,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846798) roll requested 2024-11-14T14:00:46,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846804 2024-11-14T14:00:46,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846804 2024-11-14T14:00:46,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846804) roll requested 2024-11-14T14:00:46,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846810 2024-11-14T14:00:46,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846810 2024-11-14T14:00:46,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846810) roll requested 2024-11-14T14:00:46,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846817 2024-11-14T14:00:46,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846817 2024-11-14T14:00:46,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846817) roll requested 2024-11-14T14:00:46,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846823 2024-11-14T14:00:46,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846823 2024-11-14T14:00:46,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846823) roll requested 2024-11-14T14:00:46,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846829 2024-11-14T14:00:46,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846829 2024-11-14T14:00:46,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846829) roll requested 2024-11-14T14:00:46,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846835 2024-11-14T14:00:46,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846835 2024-11-14T14:00:46,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846835) roll requested 2024-11-14T14:00:46,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846841 2024-11-14T14:00:46,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846841 2024-11-14T14:00:46,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846841) roll requested 2024-11-14T14:00:46,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846847 2024-11-14T14:00:46,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846847 2024-11-14T14:00:46,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846847) roll requested 2024-11-14T14:00:46,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846854 2024-11-14T14:00:46,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846854 2024-11-14T14:00:46,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846854) roll requested 2024-11-14T14:00:46,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846860 2024-11-14T14:00:46,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846860 2024-11-14T14:00:46,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846860) roll requested 2024-11-14T14:00:46,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846866 2024-11-14T14:00:46,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846866 2024-11-14T14:00:46,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846866) roll requested 2024-11-14T14:00:46,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846872 2024-11-14T14:00:46,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846872 2024-11-14T14:00:46,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846872) roll requested 2024-11-14T14:00:46,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846878 2024-11-14T14:00:46,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846878 2024-11-14T14:00:46,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846878) roll requested 2024-11-14T14:00:46,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846884 2024-11-14T14:00:46,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846884 2024-11-14T14:00:46,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846884) roll requested 2024-11-14T14:00:46,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846890 2024-11-14T14:00:46,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846890 2024-11-14T14:00:46,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846890) roll requested 2024-11-14T14:00:46,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846896 2024-11-14T14:00:46,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846896 2024-11-14T14:00:46,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846896) roll requested 2024-11-14T14:00:46,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846902 2024-11-14T14:00:46,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846902 2024-11-14T14:00:46,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846902) roll requested 2024-11-14T14:00:46,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846908 2024-11-14T14:00:46,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846908 2024-11-14T14:00:46,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846908) roll requested 2024-11-14T14:00:46,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846914 2024-11-14T14:00:46,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846914 2024-11-14T14:00:46,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846914) roll requested 2024-11-14T14:00:46,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846921 2024-11-14T14:00:46,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846921 2024-11-14T14:00:46,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846921) roll requested 2024-11-14T14:00:46,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846927 2024-11-14T14:00:46,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846927 2024-11-14T14:00:46,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846927) roll requested 2024-11-14T14:00:46,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846933 2024-11-14T14:00:46,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846933 2024-11-14T14:00:46,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846933) roll requested 2024-11-14T14:00:46,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846939 2024-11-14T14:00:46,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846939 2024-11-14T14:00:46,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846939) roll requested 2024-11-14T14:00:46,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846945 2024-11-14T14:00:46,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846945 2024-11-14T14:00:46,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846945) roll requested 2024-11-14T14:00:46,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846952 2024-11-14T14:00:46,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846952 2024-11-14T14:00:46,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846952) roll requested 2024-11-14T14:00:46,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846958 2024-11-14T14:00:46,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846958 2024-11-14T14:00:46,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846958) roll requested 2024-11-14T14:00:46,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846969 2024-11-14T14:00:46,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846969 2024-11-14T14:00:46,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846969) roll requested 2024-11-14T14:00:46,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846979 2024-11-14T14:00:46,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846979 2024-11-14T14:00:46,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846979) roll requested 2024-11-14T14:00:46,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846985 2024-11-14T14:00:46,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846985 2024-11-14T14:00:46,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846985) roll requested 2024-11-14T14:00:46,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846991 2024-11-14T14:00:46,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:46,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846991 2024-11-14T14:00:46,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:46,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:46,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846991) roll requested 2024-11-14T14:00:46,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592846997 2024-11-14T14:00:47,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846997 2024-11-14T14:00:47,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592846997) roll requested 2024-11-14T14:00:47,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847003 2024-11-14T14:00:47,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592846997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847003 2024-11-14T14:00:47,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847003) roll requested 2024-11-14T14:00:47,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847010 2024-11-14T14:00:47,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847010 2024-11-14T14:00:47,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847010) roll requested 2024-11-14T14:00:47,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847017 2024-11-14T14:00:47,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847017 2024-11-14T14:00:47,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847017) roll requested 2024-11-14T14:00:47,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847023 2024-11-14T14:00:47,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847023 2024-11-14T14:00:47,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847023) roll requested 2024-11-14T14:00:47,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847029 2024-11-14T14:00:47,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847029 2024-11-14T14:00:47,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847029) roll requested 2024-11-14T14:00:47,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847035 2024-11-14T14:00:47,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847035 2024-11-14T14:00:47,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847035) roll requested 2024-11-14T14:00:47,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847042 2024-11-14T14:00:47,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847042 2024-11-14T14:00:47,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847042) roll requested 2024-11-14T14:00:47,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847048 2024-11-14T14:00:47,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847048 2024-11-14T14:00:47,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847048) roll requested 2024-11-14T14:00:47,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847054 2024-11-14T14:00:47,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847054 2024-11-14T14:00:47,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847054) roll requested 2024-11-14T14:00:47,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847060 2024-11-14T14:00:47,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847060 2024-11-14T14:00:47,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847060) roll requested 2024-11-14T14:00:47,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847067 2024-11-14T14:00:47,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847067 2024-11-14T14:00:47,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847067) roll requested 2024-11-14T14:00:47,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847073 2024-11-14T14:00:47,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847073 2024-11-14T14:00:47,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847073) roll requested 2024-11-14T14:00:47,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847081 2024-11-14T14:00:47,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847081 2024-11-14T14:00:47,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847081) roll requested 2024-11-14T14:00:47,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847088 2024-11-14T14:00:47,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847088 2024-11-14T14:00:47,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847088) roll requested 2024-11-14T14:00:47,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847094 2024-11-14T14:00:47,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847094 2024-11-14T14:00:47,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847094) roll requested 2024-11-14T14:00:47,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847100 2024-11-14T14:00:47,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847100 2024-11-14T14:00:47,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847100) roll requested 2024-11-14T14:00:47,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847106 2024-11-14T14:00:47,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847106 2024-11-14T14:00:47,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847106) roll requested 2024-11-14T14:00:47,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847113 2024-11-14T14:00:47,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847113 2024-11-14T14:00:47,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847113) roll requested 2024-11-14T14:00:47,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847120 2024-11-14T14:00:47,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847120 2024-11-14T14:00:47,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847120) roll requested 2024-11-14T14:00:47,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847126 2024-11-14T14:00:47,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847126 2024-11-14T14:00:47,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847126) roll requested 2024-11-14T14:00:47,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847132 2024-11-14T14:00:47,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847132 2024-11-14T14:00:47,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847132) roll requested 2024-11-14T14:00:47,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847138 2024-11-14T14:00:47,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847138 2024-11-14T14:00:47,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847138) roll requested 2024-11-14T14:00:47,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847144 2024-11-14T14:00:47,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847144 2024-11-14T14:00:47,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847144) roll requested 2024-11-14T14:00:47,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847151 2024-11-14T14:00:47,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847151 2024-11-14T14:00:47,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847151) roll requested 2024-11-14T14:00:47,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847157 2024-11-14T14:00:47,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847157 2024-11-14T14:00:47,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847157) roll requested 2024-11-14T14:00:47,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847163 2024-11-14T14:00:47,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847163 2024-11-14T14:00:47,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847163) roll requested 2024-11-14T14:00:47,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847169 2024-11-14T14:00:47,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847169 2024-11-14T14:00:47,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847169) roll requested 2024-11-14T14:00:47,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847177 2024-11-14T14:00:47,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847177 2024-11-14T14:00:47,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847177) roll requested 2024-11-14T14:00:47,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847183 2024-11-14T14:00:47,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847183 2024-11-14T14:00:47,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847183) roll requested 2024-11-14T14:00:47,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847189 2024-11-14T14:00:47,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847189 2024-11-14T14:00:47,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847189) roll requested 2024-11-14T14:00:47,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847195 2024-11-14T14:00:47,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847195 2024-11-14T14:00:47,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847195) roll requested 2024-11-14T14:00:47,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847202 2024-11-14T14:00:47,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847202 2024-11-14T14:00:47,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847202) roll requested 2024-11-14T14:00:47,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847209 2024-11-14T14:00:47,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847209 2024-11-14T14:00:47,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847209) roll requested 2024-11-14T14:00:47,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847215 2024-11-14T14:00:47,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847215 2024-11-14T14:00:47,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847215) roll requested 2024-11-14T14:00:47,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847221 2024-11-14T14:00:47,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847221 2024-11-14T14:00:47,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847221) roll requested 2024-11-14T14:00:47,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847227 2024-11-14T14:00:47,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847227 2024-11-14T14:00:47,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847227) roll requested 2024-11-14T14:00:47,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847233 2024-11-14T14:00:47,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847233 2024-11-14T14:00:47,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847233) roll requested 2024-11-14T14:00:47,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847239 2024-11-14T14:00:47,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847239 2024-11-14T14:00:47,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847239) roll requested 2024-11-14T14:00:47,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847245 2024-11-14T14:00:47,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847245 2024-11-14T14:00:47,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847245) roll requested 2024-11-14T14:00:47,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847258 2024-11-14T14:00:47,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847258 2024-11-14T14:00:47,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847258) roll requested 2024-11-14T14:00:47,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847265 2024-11-14T14:00:47,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847265 2024-11-14T14:00:47,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847265) roll requested 2024-11-14T14:00:47,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847271 2024-11-14T14:00:47,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847271 2024-11-14T14:00:47,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847271) roll requested 2024-11-14T14:00:47,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847279 2024-11-14T14:00:47,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847279 2024-11-14T14:00:47,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847279) roll requested 2024-11-14T14:00:47,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847289 2024-11-14T14:00:47,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847289 2024-11-14T14:00:47,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847289) roll requested 2024-11-14T14:00:47,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847295 2024-11-14T14:00:47,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847295 2024-11-14T14:00:47,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847295) roll requested 2024-11-14T14:00:47,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847304 2024-11-14T14:00:47,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847304 2024-11-14T14:00:47,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847304) roll requested 2024-11-14T14:00:47,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847313 2024-11-14T14:00:47,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847313 2024-11-14T14:00:47,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847313) roll requested 2024-11-14T14:00:47,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847321 2024-11-14T14:00:47,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847321 2024-11-14T14:00:47,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847321) roll requested 2024-11-14T14:00:47,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847327 2024-11-14T14:00:47,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847327 2024-11-14T14:00:47,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847327) roll requested 2024-11-14T14:00:47,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847337 2024-11-14T14:00:47,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847337 2024-11-14T14:00:47,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847337) roll requested 2024-11-14T14:00:47,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847343 2024-11-14T14:00:47,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847343 2024-11-14T14:00:47,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847343) roll requested 2024-11-14T14:00:47,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847350 2024-11-14T14:00:47,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847350 2024-11-14T14:00:47,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847350) roll requested 2024-11-14T14:00:47,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847359 2024-11-14T14:00:47,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847359 2024-11-14T14:00:47,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847359) roll requested 2024-11-14T14:00:47,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847365 2024-11-14T14:00:47,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847365 2024-11-14T14:00:47,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847365) roll requested 2024-11-14T14:00:47,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847372 2024-11-14T14:00:47,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847372 2024-11-14T14:00:47,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847372) roll requested 2024-11-14T14:00:47,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847379 2024-11-14T14:00:47,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847379 2024-11-14T14:00:47,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847379) roll requested 2024-11-14T14:00:47,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847386 2024-11-14T14:00:47,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847386 2024-11-14T14:00:47,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847386) roll requested 2024-11-14T14:00:47,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847393 2024-11-14T14:00:47,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847393 2024-11-14T14:00:47,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847393) roll requested 2024-11-14T14:00:47,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847399 2024-11-14T14:00:47,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847399 2024-11-14T14:00:47,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847399) roll requested 2024-11-14T14:00:47,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847405 2024-11-14T14:00:47,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847405 2024-11-14T14:00:47,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847405) roll requested 2024-11-14T14:00:47,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847412 2024-11-14T14:00:47,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847412 2024-11-14T14:00:47,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847412) roll requested 2024-11-14T14:00:47,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847422 2024-11-14T14:00:47,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847422 2024-11-14T14:00:47,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847422) roll requested 2024-11-14T14:00:47,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847431 2024-11-14T14:00:47,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847431 2024-11-14T14:00:47,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847431) roll requested 2024-11-14T14:00:47,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847437 2024-11-14T14:00:47,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847437 2024-11-14T14:00:47,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847437) roll requested 2024-11-14T14:00:47,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847445 2024-11-14T14:00:47,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847445 2024-11-14T14:00:47,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847445) roll requested 2024-11-14T14:00:47,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847452 2024-11-14T14:00:47,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847452 2024-11-14T14:00:47,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847452) roll requested 2024-11-14T14:00:47,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847464 2024-11-14T14:00:47,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847464 2024-11-14T14:00:47,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847464) roll requested 2024-11-14T14:00:47,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847470 2024-11-14T14:00:47,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847470 2024-11-14T14:00:47,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847470) roll requested 2024-11-14T14:00:47,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847477 2024-11-14T14:00:47,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847477 2024-11-14T14:00:47,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847477) roll requested 2024-11-14T14:00:47,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847483 2024-11-14T14:00:47,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847483 2024-11-14T14:00:47,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847483) roll requested 2024-11-14T14:00:47,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847489 2024-11-14T14:00:47,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847489 2024-11-14T14:00:47,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847489) roll requested 2024-11-14T14:00:47,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847495 2024-11-14T14:00:47,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847495 2024-11-14T14:00:47,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847495) roll requested 2024-11-14T14:00:47,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847502 2024-11-14T14:00:47,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847502 2024-11-14T14:00:47,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847502) roll requested 2024-11-14T14:00:47,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847508 2024-11-14T14:00:47,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847508 2024-11-14T14:00:47,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847508) roll requested 2024-11-14T14:00:47,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847514 2024-11-14T14:00:47,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847514 2024-11-14T14:00:47,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847514) roll requested 2024-11-14T14:00:47,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847521 2024-11-14T14:00:47,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847521 2024-11-14T14:00:47,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847521) roll requested 2024-11-14T14:00:47,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847527 2024-11-14T14:00:47,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847527 2024-11-14T14:00:47,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847527) roll requested 2024-11-14T14:00:47,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847533 2024-11-14T14:00:47,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847533 2024-11-14T14:00:47,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847533) roll requested 2024-11-14T14:00:47,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847539 2024-11-14T14:00:47,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847539 2024-11-14T14:00:47,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847539) roll requested 2024-11-14T14:00:47,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847546 2024-11-14T14:00:47,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847546 2024-11-14T14:00:47,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847546) roll requested 2024-11-14T14:00:47,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847553 2024-11-14T14:00:47,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847553 2024-11-14T14:00:47,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847553) roll requested 2024-11-14T14:00:47,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847559 2024-11-14T14:00:47,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847559 2024-11-14T14:00:47,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847559) roll requested 2024-11-14T14:00:47,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847565 2024-11-14T14:00:47,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847565 2024-11-14T14:00:47,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847565) roll requested 2024-11-14T14:00:47,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847571 2024-11-14T14:00:47,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847571 2024-11-14T14:00:47,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847571) roll requested 2024-11-14T14:00:47,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847578 2024-11-14T14:00:47,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847578 2024-11-14T14:00:47,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847578) roll requested 2024-11-14T14:00:47,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847584 2024-11-14T14:00:47,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847584 2024-11-14T14:00:47,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847584) roll requested 2024-11-14T14:00:47,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847590 2024-11-14T14:00:47,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847590 2024-11-14T14:00:47,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847590) roll requested 2024-11-14T14:00:47,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847597 2024-11-14T14:00:47,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847597 2024-11-14T14:00:47,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847597) roll requested 2024-11-14T14:00:47,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847604 2024-11-14T14:00:47,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847604 2024-11-14T14:00:47,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847604) roll requested 2024-11-14T14:00:47,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847612 2024-11-14T14:00:47,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847612 2024-11-14T14:00:47,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847612) roll requested 2024-11-14T14:00:47,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847621 2024-11-14T14:00:47,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847621 2024-11-14T14:00:47,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847621) roll requested 2024-11-14T14:00:47,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847628 2024-11-14T14:00:47,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847628 2024-11-14T14:00:47,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847628) roll requested 2024-11-14T14:00:47,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847634 2024-11-14T14:00:47,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847634 2024-11-14T14:00:47,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847634) roll requested 2024-11-14T14:00:47,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847641 2024-11-14T14:00:47,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847641 2024-11-14T14:00:47,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847641) roll requested 2024-11-14T14:00:47,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847649 2024-11-14T14:00:47,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847649 2024-11-14T14:00:47,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847649) roll requested 2024-11-14T14:00:47,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847655 2024-11-14T14:00:47,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847655 2024-11-14T14:00:47,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847655) roll requested 2024-11-14T14:00:47,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847662 2024-11-14T14:00:47,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847662 2024-11-14T14:00:47,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847662) roll requested 2024-11-14T14:00:47,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847669 2024-11-14T14:00:47,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847669 2024-11-14T14:00:47,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847669) roll requested 2024-11-14T14:00:47,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847675 2024-11-14T14:00:47,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847675 2024-11-14T14:00:47,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847675) roll requested 2024-11-14T14:00:47,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847682 2024-11-14T14:00:47,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847682 2024-11-14T14:00:47,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847682) roll requested 2024-11-14T14:00:47,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847689 2024-11-14T14:00:47,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847689 2024-11-14T14:00:47,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847689) roll requested 2024-11-14T14:00:47,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847696 2024-11-14T14:00:47,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847696 2024-11-14T14:00:47,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847696) roll requested 2024-11-14T14:00:47,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847702 2024-11-14T14:00:47,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847702 2024-11-14T14:00:47,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847702) roll requested 2024-11-14T14:00:47,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847710 2024-11-14T14:00:47,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847710 2024-11-14T14:00:47,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847710) roll requested 2024-11-14T14:00:47,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847717 2024-11-14T14:00:47,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847717 2024-11-14T14:00:47,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847717) roll requested 2024-11-14T14:00:47,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847729 2024-11-14T14:00:47,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847729 2024-11-14T14:00:47,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847729) roll requested 2024-11-14T14:00:47,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847736 2024-11-14T14:00:47,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847736 2024-11-14T14:00:47,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847736) roll requested 2024-11-14T14:00:47,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847742 2024-11-14T14:00:47,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847742 2024-11-14T14:00:47,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847742) roll requested 2024-11-14T14:00:47,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847750 2024-11-14T14:00:47,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847750 2024-11-14T14:00:47,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847750) roll requested 2024-11-14T14:00:47,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847756 2024-11-14T14:00:47,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847756 2024-11-14T14:00:47,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847756) roll requested 2024-11-14T14:00:47,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847763 2024-11-14T14:00:47,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847763 2024-11-14T14:00:47,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847763) roll requested 2024-11-14T14:00:47,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847770 2024-11-14T14:00:47,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847770 2024-11-14T14:00:47,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847770) roll requested 2024-11-14T14:00:47,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847777 2024-11-14T14:00:47,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847777 2024-11-14T14:00:47,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847777) roll requested 2024-11-14T14:00:47,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847784 2024-11-14T14:00:47,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847784 2024-11-14T14:00:47,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847784) roll requested 2024-11-14T14:00:47,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847791 2024-11-14T14:00:47,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847791 2024-11-14T14:00:47,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847791) roll requested 2024-11-14T14:00:47,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847798 2024-11-14T14:00:47,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847798 2024-11-14T14:00:47,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847798) roll requested 2024-11-14T14:00:47,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847805 2024-11-14T14:00:47,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847805 2024-11-14T14:00:47,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847805) roll requested 2024-11-14T14:00:47,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847812 2024-11-14T14:00:47,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847812 2024-11-14T14:00:47,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847812) roll requested 2024-11-14T14:00:47,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847819 2024-11-14T14:00:47,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847819 2024-11-14T14:00:47,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847819) roll requested 2024-11-14T14:00:47,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847825 2024-11-14T14:00:47,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847825 2024-11-14T14:00:47,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847825) roll requested 2024-11-14T14:00:47,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847832 2024-11-14T14:00:47,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847832 2024-11-14T14:00:47,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847832) roll requested 2024-11-14T14:00:47,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847839 2024-11-14T14:00:47,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847839 2024-11-14T14:00:47,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847839) roll requested 2024-11-14T14:00:47,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847846 2024-11-14T14:00:47,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847846 2024-11-14T14:00:47,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847846) roll requested 2024-11-14T14:00:47,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847853 2024-11-14T14:00:47,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847853 2024-11-14T14:00:47,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847853) roll requested 2024-11-14T14:00:47,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847860 2024-11-14T14:00:47,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847860 2024-11-14T14:00:47,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847860) roll requested 2024-11-14T14:00:47,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847867 2024-11-14T14:00:47,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847867 2024-11-14T14:00:47,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847867) roll requested 2024-11-14T14:00:47,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847874 2024-11-14T14:00:47,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847874 2024-11-14T14:00:47,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847874) roll requested 2024-11-14T14:00:47,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847880 2024-11-14T14:00:47,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847880 2024-11-14T14:00:47,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847880) roll requested 2024-11-14T14:00:47,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847887 2024-11-14T14:00:47,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847887 2024-11-14T14:00:47,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847887) roll requested 2024-11-14T14:00:47,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847894 2024-11-14T14:00:47,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847894 2024-11-14T14:00:47,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847894) roll requested 2024-11-14T14:00:47,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847905 2024-11-14T14:00:47,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847905 2024-11-14T14:00:47,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847905) roll requested 2024-11-14T14:00:47,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847912 2024-11-14T14:00:47,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847912 2024-11-14T14:00:47,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847912) roll requested 2024-11-14T14:00:47,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847919 2024-11-14T14:00:47,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847919 2024-11-14T14:00:47,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847919) roll requested 2024-11-14T14:00:47,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847926 2024-11-14T14:00:47,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847926 2024-11-14T14:00:47,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847926) roll requested 2024-11-14T14:00:47,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847932 2024-11-14T14:00:47,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847932 2024-11-14T14:00:47,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847932) roll requested 2024-11-14T14:00:47,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847939 2024-11-14T14:00:47,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847939 2024-11-14T14:00:47,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847939) roll requested 2024-11-14T14:00:47,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847945 2024-11-14T14:00:47,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847945 2024-11-14T14:00:47,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847945) roll requested 2024-11-14T14:00:47,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847954 2024-11-14T14:00:47,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847954 2024-11-14T14:00:47,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847954) roll requested 2024-11-14T14:00:47,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847962 2024-11-14T14:00:47,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847962 2024-11-14T14:00:47,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847962) roll requested 2024-11-14T14:00:47,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847968 2024-11-14T14:00:47,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847968 2024-11-14T14:00:47,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847968) roll requested 2024-11-14T14:00:47,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847975 2024-11-14T14:00:47,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847975 2024-11-14T14:00:47,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847975) roll requested 2024-11-14T14:00:47,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847982 2024-11-14T14:00:47,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847982 2024-11-14T14:00:47,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:47,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847982) roll requested 2024-11-14T14:00:47,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592847993 2024-11-14T14:00:47,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:47,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847993 2024-11-14T14:00:47,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:47,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592847993) roll requested 2024-11-14T14:00:48,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848000 2024-11-14T14:00:48,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592847993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848000 2024-11-14T14:00:48,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848000) roll requested 2024-11-14T14:00:48,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848006 2024-11-14T14:00:48,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848006 2024-11-14T14:00:48,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848006) roll requested 2024-11-14T14:00:48,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848013 2024-11-14T14:00:48,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848013 2024-11-14T14:00:48,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848013) roll requested 2024-11-14T14:00:48,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848020 2024-11-14T14:00:48,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848020 2024-11-14T14:00:48,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848020) roll requested 2024-11-14T14:00:48,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848027 2024-11-14T14:00:48,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848027 2024-11-14T14:00:48,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848027) roll requested 2024-11-14T14:00:48,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848037 2024-11-14T14:00:48,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848037 2024-11-14T14:00:48,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848037) roll requested 2024-11-14T14:00:48,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848044 2024-11-14T14:00:48,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848044 2024-11-14T14:00:48,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848044) roll requested 2024-11-14T14:00:48,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848051 2024-11-14T14:00:48,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848051 2024-11-14T14:00:48,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848051) roll requested 2024-11-14T14:00:48,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848057 2024-11-14T14:00:48,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848057 2024-11-14T14:00:48,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848057) roll requested 2024-11-14T14:00:48,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848064 2024-11-14T14:00:48,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848064 2024-11-14T14:00:48,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848064) roll requested 2024-11-14T14:00:48,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848072 2024-11-14T14:00:48,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848072 2024-11-14T14:00:48,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848072) roll requested 2024-11-14T14:00:48,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848078 2024-11-14T14:00:48,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848078 2024-11-14T14:00:48,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848078) roll requested 2024-11-14T14:00:48,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848085 2024-11-14T14:00:48,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848085 2024-11-14T14:00:48,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848085) roll requested 2024-11-14T14:00:48,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848091 2024-11-14T14:00:48,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848091 2024-11-14T14:00:48,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848091) roll requested 2024-11-14T14:00:48,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848098 2024-11-14T14:00:48,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848098 2024-11-14T14:00:48,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848098) roll requested 2024-11-14T14:00:48,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848105 2024-11-14T14:00:48,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848105 2024-11-14T14:00:48,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848105) roll requested 2024-11-14T14:00:48,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848112 2024-11-14T14:00:48,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848112 2024-11-14T14:00:48,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848112) roll requested 2024-11-14T14:00:48,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848118 2024-11-14T14:00:48,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848118 2024-11-14T14:00:48,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848118) roll requested 2024-11-14T14:00:48,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848125 2024-11-14T14:00:48,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848125 2024-11-14T14:00:48,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848125) roll requested 2024-11-14T14:00:48,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848132 2024-11-14T14:00:48,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848132 2024-11-14T14:00:48,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848132) roll requested 2024-11-14T14:00:48,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848139 2024-11-14T14:00:48,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848139 2024-11-14T14:00:48,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848139) roll requested 2024-11-14T14:00:48,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848146 2024-11-14T14:00:48,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848146 2024-11-14T14:00:48,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848146) roll requested 2024-11-14T14:00:48,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848153 2024-11-14T14:00:48,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848153 2024-11-14T14:00:48,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848153) roll requested 2024-11-14T14:00:48,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848160 2024-11-14T14:00:48,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848160 2024-11-14T14:00:48,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848160) roll requested 2024-11-14T14:00:48,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848167 2024-11-14T14:00:48,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848167 2024-11-14T14:00:48,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848167) roll requested 2024-11-14T14:00:48,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848174 2024-11-14T14:00:48,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848174 2024-11-14T14:00:48,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848174) roll requested 2024-11-14T14:00:48,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848180 2024-11-14T14:00:48,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848180 2024-11-14T14:00:48,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848180) roll requested 2024-11-14T14:00:48,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848197 2024-11-14T14:00:48,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848197 2024-11-14T14:00:48,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848197) roll requested 2024-11-14T14:00:48,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848205 2024-11-14T14:00:48,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848205 2024-11-14T14:00:48,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848205) roll requested 2024-11-14T14:00:48,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848212 2024-11-14T14:00:48,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848212 2024-11-14T14:00:48,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848212) roll requested 2024-11-14T14:00:48,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848219 2024-11-14T14:00:48,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848219 2024-11-14T14:00:48,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848219) roll requested 2024-11-14T14:00:48,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848226 2024-11-14T14:00:48,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848226 2024-11-14T14:00:48,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848226) roll requested 2024-11-14T14:00:48,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848232 2024-11-14T14:00:48,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848232 2024-11-14T14:00:48,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848232) roll requested 2024-11-14T14:00:48,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848240 2024-11-14T14:00:48,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848240 2024-11-14T14:00:48,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848240) roll requested 2024-11-14T14:00:48,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848247 2024-11-14T14:00:48,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848247 2024-11-14T14:00:48,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848247) roll requested 2024-11-14T14:00:48,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848254 2024-11-14T14:00:48,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848254 2024-11-14T14:00:48,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848254) roll requested 2024-11-14T14:00:48,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848261 2024-11-14T14:00:48,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848261 2024-11-14T14:00:48,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848261) roll requested 2024-11-14T14:00:48,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848268 2024-11-14T14:00:48,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848268 2024-11-14T14:00:48,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848268) roll requested 2024-11-14T14:00:48,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848275 2024-11-14T14:00:48,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848275 2024-11-14T14:00:48,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848275) roll requested 2024-11-14T14:00:48,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848282 2024-11-14T14:00:48,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848282 2024-11-14T14:00:48,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848282) roll requested 2024-11-14T14:00:48,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848289 2024-11-14T14:00:48,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848289 2024-11-14T14:00:48,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848289) roll requested 2024-11-14T14:00:48,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848295 2024-11-14T14:00:48,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848295 2024-11-14T14:00:48,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848295) roll requested 2024-11-14T14:00:48,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848302 2024-11-14T14:00:48,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848302 2024-11-14T14:00:48,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848302) roll requested 2024-11-14T14:00:48,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848308 2024-11-14T14:00:48,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848308 2024-11-14T14:00:48,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848308) roll requested 2024-11-14T14:00:48,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848315 2024-11-14T14:00:48,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848315 2024-11-14T14:00:48,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848315) roll requested 2024-11-14T14:00:48,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848321 2024-11-14T14:00:48,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848321 2024-11-14T14:00:48,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848321) roll requested 2024-11-14T14:00:48,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848329 2024-11-14T14:00:48,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848329 2024-11-14T14:00:48,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848329) roll requested 2024-11-14T14:00:48,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848336 2024-11-14T14:00:48,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848336 2024-11-14T14:00:48,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848336) roll requested 2024-11-14T14:00:48,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848343 2024-11-14T14:00:48,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848343 2024-11-14T14:00:48,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848343) roll requested 2024-11-14T14:00:48,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848350 2024-11-14T14:00:48,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848350 2024-11-14T14:00:48,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848350) roll requested 2024-11-14T14:00:48,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848357 2024-11-14T14:00:48,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848357 2024-11-14T14:00:48,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848357) roll requested 2024-11-14T14:00:48,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848364 2024-11-14T14:00:48,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848364 2024-11-14T14:00:48,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848364) roll requested 2024-11-14T14:00:48,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848370 2024-11-14T14:00:48,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848370 2024-11-14T14:00:48,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848370) roll requested 2024-11-14T14:00:48,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848377 2024-11-14T14:00:48,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848377 2024-11-14T14:00:48,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848377) roll requested 2024-11-14T14:00:48,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848384 2024-11-14T14:00:48,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848384 2024-11-14T14:00:48,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848384) roll requested 2024-11-14T14:00:48,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848391 2024-11-14T14:00:48,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848391 2024-11-14T14:00:48,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848391) roll requested 2024-11-14T14:00:48,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848398 2024-11-14T14:00:48,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848398 2024-11-14T14:00:48,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848398) roll requested 2024-11-14T14:00:48,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848405 2024-11-14T14:00:48,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848405 2024-11-14T14:00:48,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848405) roll requested 2024-11-14T14:00:48,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848412 2024-11-14T14:00:48,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848412 2024-11-14T14:00:48,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848412) roll requested 2024-11-14T14:00:48,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848419 2024-11-14T14:00:48,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848419 2024-11-14T14:00:48,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848419) roll requested 2024-11-14T14:00:48,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848426 2024-11-14T14:00:48,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848426 2024-11-14T14:00:48,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848426) roll requested 2024-11-14T14:00:48,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848433 2024-11-14T14:00:48,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848433 2024-11-14T14:00:48,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848433) roll requested 2024-11-14T14:00:48,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848441 2024-11-14T14:00:48,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848441 2024-11-14T14:00:48,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848441) roll requested 2024-11-14T14:00:48,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848449 2024-11-14T14:00:48,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848449 2024-11-14T14:00:48,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848449) roll requested 2024-11-14T14:00:48,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848456 2024-11-14T14:00:48,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848456 2024-11-14T14:00:48,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848456) roll requested 2024-11-14T14:00:48,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848464 2024-11-14T14:00:48,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848464 2024-11-14T14:00:48,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848464) roll requested 2024-11-14T14:00:48,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848470 2024-11-14T14:00:48,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848470 2024-11-14T14:00:48,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848470) roll requested 2024-11-14T14:00:48,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848477 2024-11-14T14:00:48,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848477 2024-11-14T14:00:48,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848477) roll requested 2024-11-14T14:00:48,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848483 2024-11-14T14:00:48,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848483 2024-11-14T14:00:48,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848483) roll requested 2024-11-14T14:00:48,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848490 2024-11-14T14:00:48,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848490 2024-11-14T14:00:48,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848490) roll requested 2024-11-14T14:00:48,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848496 2024-11-14T14:00:48,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848496 2024-11-14T14:00:48,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848496) roll requested 2024-11-14T14:00:48,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848503 2024-11-14T14:00:48,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848503 2024-11-14T14:00:48,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848503) roll requested 2024-11-14T14:00:48,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848510 2024-11-14T14:00:48,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848510 2024-11-14T14:00:48,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848510) roll requested 2024-11-14T14:00:48,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848517 2024-11-14T14:00:48,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848517 2024-11-14T14:00:48,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848517) roll requested 2024-11-14T14:00:48,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848524 2024-11-14T14:00:48,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848524 2024-11-14T14:00:48,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848524) roll requested 2024-11-14T14:00:48,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848531 2024-11-14T14:00:48,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848531 2024-11-14T14:00:48,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848531) roll requested 2024-11-14T14:00:48,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848540 2024-11-14T14:00:48,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848540 2024-11-14T14:00:48,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848540) roll requested 2024-11-14T14:00:48,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848553 2024-11-14T14:00:48,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848553 2024-11-14T14:00:48,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848553) roll requested 2024-11-14T14:00:48,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848565 2024-11-14T14:00:48,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848565 2024-11-14T14:00:48,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848565) roll requested 2024-11-14T14:00:48,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848573 2024-11-14T14:00:48,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848573 2024-11-14T14:00:48,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848573) roll requested 2024-11-14T14:00:48,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848580 2024-11-14T14:00:48,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848580 2024-11-14T14:00:48,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848580) roll requested 2024-11-14T14:00:48,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848587 2024-11-14T14:00:48,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848587 2024-11-14T14:00:48,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848587) roll requested 2024-11-14T14:00:48,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848594 2024-11-14T14:00:48,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848594 2024-11-14T14:00:48,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848594) roll requested 2024-11-14T14:00:48,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848601 2024-11-14T14:00:48,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848601 2024-11-14T14:00:48,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848601) roll requested 2024-11-14T14:00:48,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848608 2024-11-14T14:00:48,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848608 2024-11-14T14:00:48,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848608) roll requested 2024-11-14T14:00:48,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848615 2024-11-14T14:00:48,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848615 2024-11-14T14:00:48,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848615) roll requested 2024-11-14T14:00:48,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848622 2024-11-14T14:00:48,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848622 2024-11-14T14:00:48,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848622) roll requested 2024-11-14T14:00:48,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848628 2024-11-14T14:00:48,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848628 2024-11-14T14:00:48,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848628) roll requested 2024-11-14T14:00:48,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848635 2024-11-14T14:00:48,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848635 2024-11-14T14:00:48,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848635) roll requested 2024-11-14T14:00:48,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848645 2024-11-14T14:00:48,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848645 2024-11-14T14:00:48,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848645) roll requested 2024-11-14T14:00:48,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848652 2024-11-14T14:00:48,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848652 2024-11-14T14:00:48,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848652) roll requested 2024-11-14T14:00:48,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848659 2024-11-14T14:00:48,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848659 2024-11-14T14:00:48,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848659) roll requested 2024-11-14T14:00:48,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848666 2024-11-14T14:00:48,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848666 2024-11-14T14:00:48,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848666) roll requested 2024-11-14T14:00:48,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848672 2024-11-14T14:00:48,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848672 2024-11-14T14:00:48,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848672) roll requested 2024-11-14T14:00:48,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848679 2024-11-14T14:00:48,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848679 2024-11-14T14:00:48,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848679) roll requested 2024-11-14T14:00:48,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848686 2024-11-14T14:00:48,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848686 2024-11-14T14:00:48,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848686) roll requested 2024-11-14T14:00:48,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848692 2024-11-14T14:00:48,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848692 2024-11-14T14:00:48,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848692) roll requested 2024-11-14T14:00:48,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848699 2024-11-14T14:00:48,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848699 2024-11-14T14:00:48,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848699) roll requested 2024-11-14T14:00:48,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848705 2024-11-14T14:00:48,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848705 2024-11-14T14:00:48,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848705) roll requested 2024-11-14T14:00:48,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848712 2024-11-14T14:00:48,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848712 2024-11-14T14:00:48,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848712) roll requested 2024-11-14T14:00:48,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848719 2024-11-14T14:00:48,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848719 2024-11-14T14:00:48,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848719) roll requested 2024-11-14T14:00:48,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848725 2024-11-14T14:00:48,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848725 2024-11-14T14:00:48,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848725) roll requested 2024-11-14T14:00:48,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848732 2024-11-14T14:00:48,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848732 2024-11-14T14:00:48,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848732) roll requested 2024-11-14T14:00:48,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848740 2024-11-14T14:00:48,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848740 2024-11-14T14:00:48,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848740) roll requested 2024-11-14T14:00:48,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848747 2024-11-14T14:00:48,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848747 2024-11-14T14:00:48,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848747) roll requested 2024-11-14T14:00:48,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848754 2024-11-14T14:00:48,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848754 2024-11-14T14:00:48,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848754) roll requested 2024-11-14T14:00:48,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848761 2024-11-14T14:00:48,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848761 2024-11-14T14:00:48,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848761) roll requested 2024-11-14T14:00:48,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848768 2024-11-14T14:00:48,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848768 2024-11-14T14:00:48,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848768) roll requested 2024-11-14T14:00:48,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848775 2024-11-14T14:00:48,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848775 2024-11-14T14:00:48,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848775) roll requested 2024-11-14T14:00:48,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848781 2024-11-14T14:00:48,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848781 2024-11-14T14:00:48,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848781) roll requested 2024-11-14T14:00:48,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848788 2024-11-14T14:00:48,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848788 2024-11-14T14:00:48,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848788) roll requested 2024-11-14T14:00:48,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848795 2024-11-14T14:00:48,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848795 2024-11-14T14:00:48,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848795) roll requested 2024-11-14T14:00:48,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848801 2024-11-14T14:00:48,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848801 2024-11-14T14:00:48,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848801) roll requested 2024-11-14T14:00:48,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848808 2024-11-14T14:00:48,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848808 2024-11-14T14:00:48,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848808) roll requested 2024-11-14T14:00:48,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848815 2024-11-14T14:00:48,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848815 2024-11-14T14:00:48,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848815) roll requested 2024-11-14T14:00:48,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848822 2024-11-14T14:00:48,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848822 2024-11-14T14:00:48,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848822) roll requested 2024-11-14T14:00:48,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848828 2024-11-14T14:00:48,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848828 2024-11-14T14:00:48,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848828) roll requested 2024-11-14T14:00:48,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848835 2024-11-14T14:00:48,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848835 2024-11-14T14:00:48,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848835) roll requested 2024-11-14T14:00:48,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848842 2024-11-14T14:00:48,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848842 2024-11-14T14:00:48,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848842) roll requested 2024-11-14T14:00:48,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848849 2024-11-14T14:00:48,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848849 2024-11-14T14:00:48,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848849) roll requested 2024-11-14T14:00:48,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848856 2024-11-14T14:00:48,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848856 2024-11-14T14:00:48,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848856) roll requested 2024-11-14T14:00:48,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848863 2024-11-14T14:00:48,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848863 2024-11-14T14:00:48,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848863) roll requested 2024-11-14T14:00:48,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848869 2024-11-14T14:00:48,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848869 2024-11-14T14:00:48,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848869) roll requested 2024-11-14T14:00:48,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848876 2024-11-14T14:00:48,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848876 2024-11-14T14:00:48,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848876) roll requested 2024-11-14T14:00:48,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848883 2024-11-14T14:00:48,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848883 2024-11-14T14:00:48,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848883) roll requested 2024-11-14T14:00:48,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848890 2024-11-14T14:00:48,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848890 2024-11-14T14:00:48,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848890) roll requested 2024-11-14T14:00:48,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848896 2024-11-14T14:00:48,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848896 2024-11-14T14:00:48,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848896) roll requested 2024-11-14T14:00:48,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848903 2024-11-14T14:00:48,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848903 2024-11-14T14:00:48,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848903) roll requested 2024-11-14T14:00:48,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848909 2024-11-14T14:00:48,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848909 2024-11-14T14:00:48,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848909) roll requested 2024-11-14T14:00:48,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848915 2024-11-14T14:00:48,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848915 2024-11-14T14:00:48,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848915) roll requested 2024-11-14T14:00:48,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848922 2024-11-14T14:00:48,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848922 2024-11-14T14:00:48,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848922) roll requested 2024-11-14T14:00:48,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848928 2024-11-14T14:00:48,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848928 2024-11-14T14:00:48,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848928) roll requested 2024-11-14T14:00:48,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848935 2024-11-14T14:00:48,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848935 2024-11-14T14:00:48,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848935) roll requested 2024-11-14T14:00:48,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848942 2024-11-14T14:00:48,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848942 2024-11-14T14:00:48,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848942) roll requested 2024-11-14T14:00:48,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848948 2024-11-14T14:00:48,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848948 2024-11-14T14:00:48,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848948) roll requested 2024-11-14T14:00:48,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848955 2024-11-14T14:00:48,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848955 2024-11-14T14:00:48,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848955) roll requested 2024-11-14T14:00:48,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848961 2024-11-14T14:00:48,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848961 2024-11-14T14:00:48,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848961) roll requested 2024-11-14T14:00:48,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848968 2024-11-14T14:00:48,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848968 2024-11-14T14:00:48,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848968) roll requested 2024-11-14T14:00:48,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848974 2024-11-14T14:00:48,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848974 2024-11-14T14:00:48,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848974) roll requested 2024-11-14T14:00:48,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848981 2024-11-14T14:00:48,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848981 2024-11-14T14:00:48,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848981) roll requested 2024-11-14T14:00:48,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848988 2024-11-14T14:00:48,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:48,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848988 2024-11-14T14:00:48,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:48,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:48,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848988) roll requested 2024-11-14T14:00:48,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592848994 2024-11-14T14:00:49,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848994 2024-11-14T14:00:49,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592848994) roll requested 2024-11-14T14:00:49,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849001 2024-11-14T14:00:49,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592848994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849001 2024-11-14T14:00:49,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849001) roll requested 2024-11-14T14:00:49,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849008 2024-11-14T14:00:49,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849008 2024-11-14T14:00:49,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849008) roll requested 2024-11-14T14:00:49,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849014 2024-11-14T14:00:49,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849014 2024-11-14T14:00:49,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849014) roll requested 2024-11-14T14:00:49,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849020 2024-11-14T14:00:49,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849020 2024-11-14T14:00:49,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849020) roll requested 2024-11-14T14:00:49,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849027 2024-11-14T14:00:49,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849027 2024-11-14T14:00:49,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849027) roll requested 2024-11-14T14:00:49,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849034 2024-11-14T14:00:49,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849034 2024-11-14T14:00:49,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849034) roll requested 2024-11-14T14:00:49,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849041 2024-11-14T14:00:49,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849041 2024-11-14T14:00:49,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849041) roll requested 2024-11-14T14:00:49,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849048 2024-11-14T14:00:49,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849048 2024-11-14T14:00:49,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849048) roll requested 2024-11-14T14:00:49,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849055 2024-11-14T14:00:49,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849055 2024-11-14T14:00:49,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849055) roll requested 2024-11-14T14:00:49,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849062 2024-11-14T14:00:49,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849062 2024-11-14T14:00:49,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849062) roll requested 2024-11-14T14:00:49,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849068 2024-11-14T14:00:49,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849068 2024-11-14T14:00:49,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849068) roll requested 2024-11-14T14:00:49,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849075 2024-11-14T14:00:49,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849075 2024-11-14T14:00:49,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849075) roll requested 2024-11-14T14:00:49,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849082 2024-11-14T14:00:49,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849082 2024-11-14T14:00:49,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849082) roll requested 2024-11-14T14:00:49,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849089 2024-11-14T14:00:49,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849089 2024-11-14T14:00:49,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849089) roll requested 2024-11-14T14:00:49,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849096 2024-11-14T14:00:49,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849096 2024-11-14T14:00:49,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849096) roll requested 2024-11-14T14:00:49,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849103 2024-11-14T14:00:49,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849103 2024-11-14T14:00:49,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849103) roll requested 2024-11-14T14:00:49,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849109 2024-11-14T14:00:49,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849109 2024-11-14T14:00:49,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849109) roll requested 2024-11-14T14:00:49,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849116 2024-11-14T14:00:49,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849116 2024-11-14T14:00:49,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849116) roll requested 2024-11-14T14:00:49,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849123 2024-11-14T14:00:49,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849123 2024-11-14T14:00:49,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849123) roll requested 2024-11-14T14:00:49,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849129 2024-11-14T14:00:49,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849129 2024-11-14T14:00:49,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849129) roll requested 2024-11-14T14:00:49,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849136 2024-11-14T14:00:49,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849136 2024-11-14T14:00:49,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849136) roll requested 2024-11-14T14:00:49,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849144 2024-11-14T14:00:49,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849144 2024-11-14T14:00:49,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849144) roll requested 2024-11-14T14:00:49,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849152 2024-11-14T14:00:49,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849152 2024-11-14T14:00:49,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849152) roll requested 2024-11-14T14:00:49,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849159 2024-11-14T14:00:49,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849159 2024-11-14T14:00:49,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849159) roll requested 2024-11-14T14:00:49,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849166 2024-11-14T14:00:49,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849166 2024-11-14T14:00:49,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849166) roll requested 2024-11-14T14:00:49,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849173 2024-11-14T14:00:49,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849173 2024-11-14T14:00:49,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849173) roll requested 2024-11-14T14:00:49,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849180 2024-11-14T14:00:49,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849180 2024-11-14T14:00:49,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849180) roll requested 2024-11-14T14:00:49,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849188 2024-11-14T14:00:49,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849188 2024-11-14T14:00:49,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849188) roll requested 2024-11-14T14:00:49,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849195 2024-11-14T14:00:49,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849195 2024-11-14T14:00:49,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849195) roll requested 2024-11-14T14:00:49,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849201 2024-11-14T14:00:49,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849201 2024-11-14T14:00:49,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849201) roll requested 2024-11-14T14:00:49,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849208 2024-11-14T14:00:49,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849208 2024-11-14T14:00:49,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849208) roll requested 2024-11-14T14:00:49,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849214 2024-11-14T14:00:49,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849214 2024-11-14T14:00:49,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849214) roll requested 2024-11-14T14:00:49,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849222 2024-11-14T14:00:49,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849222 2024-11-14T14:00:49,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849222) roll requested 2024-11-14T14:00:49,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849228 2024-11-14T14:00:49,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849228 2024-11-14T14:00:49,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849228) roll requested 2024-11-14T14:00:49,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849235 2024-11-14T14:00:49,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849235 2024-11-14T14:00:49,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849235) roll requested 2024-11-14T14:00:49,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849242 2024-11-14T14:00:49,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849242 2024-11-14T14:00:49,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849242) roll requested 2024-11-14T14:00:49,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849249 2024-11-14T14:00:49,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849249 2024-11-14T14:00:49,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849249) roll requested 2024-11-14T14:00:49,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849256 2024-11-14T14:00:49,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849256 2024-11-14T14:00:49,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849256) roll requested 2024-11-14T14:00:49,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849263 2024-11-14T14:00:49,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849263 2024-11-14T14:00:49,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849263) roll requested 2024-11-14T14:00:49,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849269 2024-11-14T14:00:49,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849269 2024-11-14T14:00:49,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849269) roll requested 2024-11-14T14:00:49,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849276 2024-11-14T14:00:49,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849276 2024-11-14T14:00:49,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849276) roll requested 2024-11-14T14:00:49,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849283 2024-11-14T14:00:49,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849283 2024-11-14T14:00:49,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849283) roll requested 2024-11-14T14:00:49,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849290 2024-11-14T14:00:49,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849290 2024-11-14T14:00:49,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849290) roll requested 2024-11-14T14:00:49,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849297 2024-11-14T14:00:49,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849297 2024-11-14T14:00:49,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849297) roll requested 2024-11-14T14:00:49,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849304 2024-11-14T14:00:49,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849304 2024-11-14T14:00:49,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849304) roll requested 2024-11-14T14:00:49,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849310 2024-11-14T14:00:49,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849310 2024-11-14T14:00:49,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849310) roll requested 2024-11-14T14:00:49,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849317 2024-11-14T14:00:49,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849317 2024-11-14T14:00:49,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849317) roll requested 2024-11-14T14:00:49,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849325 2024-11-14T14:00:49,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849325 2024-11-14T14:00:49,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849325) roll requested 2024-11-14T14:00:49,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849332 2024-11-14T14:00:49,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849332 2024-11-14T14:00:49,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849332) roll requested 2024-11-14T14:00:49,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849340 2024-11-14T14:00:49,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849340 2024-11-14T14:00:49,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849340) roll requested 2024-11-14T14:00:49,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849355 2024-11-14T14:00:49,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849355 2024-11-14T14:00:49,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849355) roll requested 2024-11-14T14:00:49,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849362 2024-11-14T14:00:49,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849362 2024-11-14T14:00:49,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849362) roll requested 2024-11-14T14:00:49,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849368 2024-11-14T14:00:49,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849368 2024-11-14T14:00:49,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849368) roll requested 2024-11-14T14:00:49,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849375 2024-11-14T14:00:49,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849375 2024-11-14T14:00:49,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849375) roll requested 2024-11-14T14:00:49,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849382 2024-11-14T14:00:49,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849382 2024-11-14T14:00:49,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849382) roll requested 2024-11-14T14:00:49,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849390 2024-11-14T14:00:49,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849390 2024-11-14T14:00:49,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849390) roll requested 2024-11-14T14:00:49,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849396 2024-11-14T14:00:49,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849396 2024-11-14T14:00:49,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849396) roll requested 2024-11-14T14:00:49,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849403 2024-11-14T14:00:49,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849403 2024-11-14T14:00:49,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849403) roll requested 2024-11-14T14:00:49,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849410 2024-11-14T14:00:49,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849410 2024-11-14T14:00:49,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849410) roll requested 2024-11-14T14:00:49,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849417 2024-11-14T14:00:49,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849417 2024-11-14T14:00:49,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849417) roll requested 2024-11-14T14:00:49,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849424 2024-11-14T14:00:49,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849424 2024-11-14T14:00:49,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849424) roll requested 2024-11-14T14:00:49,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849432 2024-11-14T14:00:49,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849432 2024-11-14T14:00:49,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849432) roll requested 2024-11-14T14:00:49,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849438 2024-11-14T14:00:49,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849438 2024-11-14T14:00:49,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849438) roll requested 2024-11-14T14:00:49,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849445 2024-11-14T14:00:49,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849445 2024-11-14T14:00:49,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849445) roll requested 2024-11-14T14:00:49,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849452 2024-11-14T14:00:49,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849452 2024-11-14T14:00:49,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849452) roll requested 2024-11-14T14:00:49,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849459 2024-11-14T14:00:49,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849459 2024-11-14T14:00:49,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849459) roll requested 2024-11-14T14:00:49,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849466 2024-11-14T14:00:49,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849466 2024-11-14T14:00:49,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849466) roll requested 2024-11-14T14:00:49,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849473 2024-11-14T14:00:49,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849473 2024-11-14T14:00:49,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849473) roll requested 2024-11-14T14:00:49,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849480 2024-11-14T14:00:49,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849480 2024-11-14T14:00:49,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849480) roll requested 2024-11-14T14:00:49,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849487 2024-11-14T14:00:49,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849487 2024-11-14T14:00:49,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849487) roll requested 2024-11-14T14:00:49,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849494 2024-11-14T14:00:49,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849494 2024-11-14T14:00:49,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849494) roll requested 2024-11-14T14:00:49,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849501 2024-11-14T14:00:49,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849501 2024-11-14T14:00:49,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849501) roll requested 2024-11-14T14:00:49,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849508 2024-11-14T14:00:49,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849508 2024-11-14T14:00:49,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849508) roll requested 2024-11-14T14:00:49,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849514 2024-11-14T14:00:49,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849514 2024-11-14T14:00:49,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849514) roll requested 2024-11-14T14:00:49,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849521 2024-11-14T14:00:49,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849521 2024-11-14T14:00:49,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849521) roll requested 2024-11-14T14:00:49,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849529 2024-11-14T14:00:49,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849529 2024-11-14T14:00:49,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849529) roll requested 2024-11-14T14:00:49,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849536 2024-11-14T14:00:49,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849536 2024-11-14T14:00:49,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849536) roll requested 2024-11-14T14:00:49,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849543 2024-11-14T14:00:49,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849543 2024-11-14T14:00:49,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849543) roll requested 2024-11-14T14:00:49,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849549 2024-11-14T14:00:49,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849549 2024-11-14T14:00:49,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849549) roll requested 2024-11-14T14:00:49,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849556 2024-11-14T14:00:49,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849556 2024-11-14T14:00:49,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849556) roll requested 2024-11-14T14:00:49,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849563 2024-11-14T14:00:49,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849563 2024-11-14T14:00:49,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849563) roll requested 2024-11-14T14:00:49,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849570 2024-11-14T14:00:49,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849570 2024-11-14T14:00:49,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849570) roll requested 2024-11-14T14:00:49,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849577 2024-11-14T14:00:49,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849577 2024-11-14T14:00:49,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849577) roll requested 2024-11-14T14:00:49,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849585 2024-11-14T14:00:49,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849585 2024-11-14T14:00:49,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849585) roll requested 2024-11-14T14:00:49,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849592 2024-11-14T14:00:49,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849592 2024-11-14T14:00:49,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849592) roll requested 2024-11-14T14:00:49,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849599 2024-11-14T14:00:49,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849599 2024-11-14T14:00:49,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849599) roll requested 2024-11-14T14:00:49,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849605 2024-11-14T14:00:49,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849605 2024-11-14T14:00:49,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849605) roll requested 2024-11-14T14:00:49,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849611 2024-11-14T14:00:49,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849611 2024-11-14T14:00:49,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849611) roll requested 2024-11-14T14:00:49,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849618 2024-11-14T14:00:49,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849618 2024-11-14T14:00:49,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849618) roll requested 2024-11-14T14:00:49,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849625 2024-11-14T14:00:49,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849625 2024-11-14T14:00:49,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849625) roll requested 2024-11-14T14:00:49,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849632 2024-11-14T14:00:49,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849632 2024-11-14T14:00:49,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849632) roll requested 2024-11-14T14:00:49,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849639 2024-11-14T14:00:49,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849639 2024-11-14T14:00:49,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849639) roll requested 2024-11-14T14:00:49,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849647 2024-11-14T14:00:49,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849647 2024-11-14T14:00:49,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849647) roll requested 2024-11-14T14:00:49,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849654 2024-11-14T14:00:49,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849654 2024-11-14T14:00:49,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849654) roll requested 2024-11-14T14:00:49,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849661 2024-11-14T14:00:49,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849661 2024-11-14T14:00:49,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849661) roll requested 2024-11-14T14:00:49,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849667 2024-11-14T14:00:49,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849667 2024-11-14T14:00:49,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849667) roll requested 2024-11-14T14:00:49,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849675 2024-11-14T14:00:49,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849675 2024-11-14T14:00:49,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849675) roll requested 2024-11-14T14:00:49,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849681 2024-11-14T14:00:49,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849681 2024-11-14T14:00:49,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849681) roll requested 2024-11-14T14:00:49,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849688 2024-11-14T14:00:49,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849688 2024-11-14T14:00:49,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849688) roll requested 2024-11-14T14:00:49,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849695 2024-11-14T14:00:49,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849695 2024-11-14T14:00:49,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849695) roll requested 2024-11-14T14:00:49,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849702 2024-11-14T14:00:49,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849702 2024-11-14T14:00:49,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849702) roll requested 2024-11-14T14:00:49,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849708 2024-11-14T14:00:49,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849708 2024-11-14T14:00:49,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849708) roll requested 2024-11-14T14:00:49,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849715 2024-11-14T14:00:49,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849715 2024-11-14T14:00:49,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849715) roll requested 2024-11-14T14:00:49,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849722 2024-11-14T14:00:49,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849722 2024-11-14T14:00:49,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849722) roll requested 2024-11-14T14:00:49,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849729 2024-11-14T14:00:49,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849729 2024-11-14T14:00:49,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849729) roll requested 2024-11-14T14:00:49,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849736 2024-11-14T14:00:49,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849736 2024-11-14T14:00:49,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849736) roll requested 2024-11-14T14:00:49,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849743 2024-11-14T14:00:49,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849743 2024-11-14T14:00:49,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849743) roll requested 2024-11-14T14:00:49,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849752 2024-11-14T14:00:49,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849752 2024-11-14T14:00:49,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849752) roll requested 2024-11-14T14:00:49,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849758 2024-11-14T14:00:49,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849758 2024-11-14T14:00:49,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849758) roll requested 2024-11-14T14:00:49,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849765 2024-11-14T14:00:49,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849765 2024-11-14T14:00:49,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849765) roll requested 2024-11-14T14:00:49,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849771 2024-11-14T14:00:49,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849771 2024-11-14T14:00:49,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849771) roll requested 2024-11-14T14:00:49,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849778 2024-11-14T14:00:49,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849778 2024-11-14T14:00:49,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849778) roll requested 2024-11-14T14:00:49,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849784 2024-11-14T14:00:49,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849784 2024-11-14T14:00:49,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849784) roll requested 2024-11-14T14:00:49,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849790 2024-11-14T14:00:49,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849790 2024-11-14T14:00:49,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849790) roll requested 2024-11-14T14:00:49,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849801 2024-11-14T14:00:49,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849801 2024-11-14T14:00:49,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849801) roll requested 2024-11-14T14:00:49,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849809 2024-11-14T14:00:49,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849809 2024-11-14T14:00:49,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849809) roll requested 2024-11-14T14:00:49,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849816 2024-11-14T14:00:49,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849816 2024-11-14T14:00:49,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849816) roll requested 2024-11-14T14:00:49,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849823 2024-11-14T14:00:49,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849823 2024-11-14T14:00:49,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849823) roll requested 2024-11-14T14:00:49,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849830 2024-11-14T14:00:49,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849830 2024-11-14T14:00:49,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849830) roll requested 2024-11-14T14:00:49,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849837 2024-11-14T14:00:49,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849837 2024-11-14T14:00:49,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849837) roll requested 2024-11-14T14:00:49,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849844 2024-11-14T14:00:49,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849844 2024-11-14T14:00:49,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849844) roll requested 2024-11-14T14:00:49,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849850 2024-11-14T14:00:49,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849850 2024-11-14T14:00:49,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849850) roll requested 2024-11-14T14:00:49,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849857 2024-11-14T14:00:49,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849857 2024-11-14T14:00:49,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849857) roll requested 2024-11-14T14:00:49,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849864 2024-11-14T14:00:49,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849864 2024-11-14T14:00:49,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849864) roll requested 2024-11-14T14:00:49,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849871 2024-11-14T14:00:49,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849871 2024-11-14T14:00:49,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849871) roll requested 2024-11-14T14:00:49,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849877 2024-11-14T14:00:49,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849877 2024-11-14T14:00:49,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849877) roll requested 2024-11-14T14:00:49,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849884 2024-11-14T14:00:49,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849884 2024-11-14T14:00:49,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849884) roll requested 2024-11-14T14:00:49,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849890 2024-11-14T14:00:49,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849890 2024-11-14T14:00:49,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849890) roll requested 2024-11-14T14:00:49,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849897 2024-11-14T14:00:49,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849897 2024-11-14T14:00:49,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849897) roll requested 2024-11-14T14:00:49,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849904 2024-11-14T14:00:49,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849904 2024-11-14T14:00:49,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849904) roll requested 2024-11-14T14:00:49,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849910 2024-11-14T14:00:49,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849910 2024-11-14T14:00:49,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849910) roll requested 2024-11-14T14:00:49,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849917 2024-11-14T14:00:49,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849917 2024-11-14T14:00:49,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849917) roll requested 2024-11-14T14:00:49,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849924 2024-11-14T14:00:49,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849924 2024-11-14T14:00:49,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849924) roll requested 2024-11-14T14:00:49,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849931 2024-11-14T14:00:49,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849931 2024-11-14T14:00:49,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849931) roll requested 2024-11-14T14:00:49,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849938 2024-11-14T14:00:49,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849938 2024-11-14T14:00:49,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849938) roll requested 2024-11-14T14:00:49,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849944 2024-11-14T14:00:49,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849944 2024-11-14T14:00:49,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849944) roll requested 2024-11-14T14:00:49,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849951 2024-11-14T14:00:49,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849951 2024-11-14T14:00:49,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849951) roll requested 2024-11-14T14:00:49,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849958 2024-11-14T14:00:49,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849958 2024-11-14T14:00:49,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849958) roll requested 2024-11-14T14:00:49,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849965 2024-11-14T14:00:49,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849965 2024-11-14T14:00:49,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849965) roll requested 2024-11-14T14:00:49,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849972 2024-11-14T14:00:49,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849972 2024-11-14T14:00:49,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849972) roll requested 2024-11-14T14:00:49,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849979 2024-11-14T14:00:49,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849979 2024-11-14T14:00:49,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849979) roll requested 2024-11-14T14:00:49,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849986 2024-11-14T14:00:49,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849986 2024-11-14T14:00:49,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849986) roll requested 2024-11-14T14:00:49,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849992 2024-11-14T14:00:49,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:49,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849992 2024-11-14T14:00:49,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:49,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:49,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849992) roll requested 2024-11-14T14:00:49,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592849999 2024-11-14T14:00:50,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849999 2024-11-14T14:00:50,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592849999) roll requested 2024-11-14T14:00:50,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850006 2024-11-14T14:00:50,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592849999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850006 2024-11-14T14:00:50,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850006) roll requested 2024-11-14T14:00:50,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850016 2024-11-14T14:00:50,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850016 2024-11-14T14:00:50,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850016) roll requested 2024-11-14T14:00:50,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850023 2024-11-14T14:00:50,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850023 2024-11-14T14:00:50,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850023) roll requested 2024-11-14T14:00:50,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850030 2024-11-14T14:00:50,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850030 2024-11-14T14:00:50,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850030) roll requested 2024-11-14T14:00:50,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850037 2024-11-14T14:00:50,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850037 2024-11-14T14:00:50,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850037) roll requested 2024-11-14T14:00:50,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850044 2024-11-14T14:00:50,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850044 2024-11-14T14:00:50,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850044) roll requested 2024-11-14T14:00:50,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850051 2024-11-14T14:00:50,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850051 2024-11-14T14:00:50,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850051) roll requested 2024-11-14T14:00:50,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850059 2024-11-14T14:00:50,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850059 2024-11-14T14:00:50,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850059) roll requested 2024-11-14T14:00:50,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850066 2024-11-14T14:00:50,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850066 2024-11-14T14:00:50,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850066) roll requested 2024-11-14T14:00:50,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850073 2024-11-14T14:00:50,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850073 2024-11-14T14:00:50,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850073) roll requested 2024-11-14T14:00:50,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850080 2024-11-14T14:00:50,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850080 2024-11-14T14:00:50,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850080) roll requested 2024-11-14T14:00:50,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850089 2024-11-14T14:00:50,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850089 2024-11-14T14:00:50,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850089) roll requested 2024-11-14T14:00:50,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850096 2024-11-14T14:00:50,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850096 2024-11-14T14:00:50,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850096) roll requested 2024-11-14T14:00:50,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850106 2024-11-14T14:00:50,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850106 2024-11-14T14:00:50,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850106) roll requested 2024-11-14T14:00:50,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850113 2024-11-14T14:00:50,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850113 2024-11-14T14:00:50,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850113) roll requested 2024-11-14T14:00:50,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850120 2024-11-14T14:00:50,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850120 2024-11-14T14:00:50,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850120) roll requested 2024-11-14T14:00:50,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850127 2024-11-14T14:00:50,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850127 2024-11-14T14:00:50,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850127) roll requested 2024-11-14T14:00:50,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850134 2024-11-14T14:00:50,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850134 2024-11-14T14:00:50,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850134) roll requested 2024-11-14T14:00:50,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850141 2024-11-14T14:00:50,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850141 2024-11-14T14:00:50,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850141) roll requested 2024-11-14T14:00:50,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850148 2024-11-14T14:00:50,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850148 2024-11-14T14:00:50,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850148) roll requested 2024-11-14T14:00:50,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850155 2024-11-14T14:00:50,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850155 2024-11-14T14:00:50,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850155) roll requested 2024-11-14T14:00:50,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850162 2024-11-14T14:00:50,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850162 2024-11-14T14:00:50,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850162) roll requested 2024-11-14T14:00:50,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850169 2024-11-14T14:00:50,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850169 2024-11-14T14:00:50,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850169) roll requested 2024-11-14T14:00:50,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850176 2024-11-14T14:00:50,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850176 2024-11-14T14:00:50,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850176) roll requested 2024-11-14T14:00:50,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850188 2024-11-14T14:00:50,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850188 2024-11-14T14:00:50,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850188) roll requested 2024-11-14T14:00:50,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850195 2024-11-14T14:00:50,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850195 2024-11-14T14:00:50,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850195) roll requested 2024-11-14T14:00:50,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850201 2024-11-14T14:00:50,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850201 2024-11-14T14:00:50,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850201) roll requested 2024-11-14T14:00:50,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850208 2024-11-14T14:00:50,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850208 2024-11-14T14:00:50,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850208) roll requested 2024-11-14T14:00:50,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850215 2024-11-14T14:00:50,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850215 2024-11-14T14:00:50,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850215) roll requested 2024-11-14T14:00:50,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850222 2024-11-14T14:00:50,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850222 2024-11-14T14:00:50,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850222) roll requested 2024-11-14T14:00:50,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850228 2024-11-14T14:00:50,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850228 2024-11-14T14:00:50,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850228) roll requested 2024-11-14T14:00:50,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850235 2024-11-14T14:00:50,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850235 2024-11-14T14:00:50,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850235) roll requested 2024-11-14T14:00:50,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850242 2024-11-14T14:00:50,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850242 2024-11-14T14:00:50,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850242) roll requested 2024-11-14T14:00:50,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850249 2024-11-14T14:00:50,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850249 2024-11-14T14:00:50,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850249) roll requested 2024-11-14T14:00:50,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850256 2024-11-14T14:00:50,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850256 2024-11-14T14:00:50,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850256) roll requested 2024-11-14T14:00:50,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850273 2024-11-14T14:00:50,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850273 2024-11-14T14:00:50,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850273) roll requested 2024-11-14T14:00:50,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850280 2024-11-14T14:00:50,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850280 2024-11-14T14:00:50,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850280) roll requested 2024-11-14T14:00:50,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850287 2024-11-14T14:00:50,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850287 2024-11-14T14:00:50,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850287) roll requested 2024-11-14T14:00:50,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850294 2024-11-14T14:00:50,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850294 2024-11-14T14:00:50,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850294) roll requested 2024-11-14T14:00:50,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850305 2024-11-14T14:00:50,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850305 2024-11-14T14:00:50,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850305) roll requested 2024-11-14T14:00:50,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850312 2024-11-14T14:00:50,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850312 2024-11-14T14:00:50,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850312) roll requested 2024-11-14T14:00:50,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850319 2024-11-14T14:00:50,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850319 2024-11-14T14:00:50,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850319) roll requested 2024-11-14T14:00:50,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850326 2024-11-14T14:00:50,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850326 2024-11-14T14:00:50,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850326) roll requested 2024-11-14T14:00:50,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850333 2024-11-14T14:00:50,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850333 2024-11-14T14:00:50,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850333) roll requested 2024-11-14T14:00:50,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850339 2024-11-14T14:00:50,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850339 2024-11-14T14:00:50,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850339) roll requested 2024-11-14T14:00:50,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850347 2024-11-14T14:00:50,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850347 2024-11-14T14:00:50,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850347) roll requested 2024-11-14T14:00:50,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850353 2024-11-14T14:00:50,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850353 2024-11-14T14:00:50,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850353) roll requested 2024-11-14T14:00:50,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850360 2024-11-14T14:00:50,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850360 2024-11-14T14:00:50,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850360) roll requested 2024-11-14T14:00:50,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850367 2024-11-14T14:00:50,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850367 2024-11-14T14:00:50,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850367) roll requested 2024-11-14T14:00:50,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850374 2024-11-14T14:00:50,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850374 2024-11-14T14:00:50,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850374) roll requested 2024-11-14T14:00:50,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850381 2024-11-14T14:00:50,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850381 2024-11-14T14:00:50,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850381) roll requested 2024-11-14T14:00:50,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850389 2024-11-14T14:00:50,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850389 2024-11-14T14:00:50,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850389) roll requested 2024-11-14T14:00:50,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850401 2024-11-14T14:00:50,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850401 2024-11-14T14:00:50,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850401) roll requested 2024-11-14T14:00:50,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850410 2024-11-14T14:00:50,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850410 2024-11-14T14:00:50,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850410) roll requested 2024-11-14T14:00:50,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850422 2024-11-14T14:00:50,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850422 2024-11-14T14:00:50,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850422) roll requested 2024-11-14T14:00:50,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850428 2024-11-14T14:00:50,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850428 2024-11-14T14:00:50,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850428) roll requested 2024-11-14T14:00:50,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850436 2024-11-14T14:00:50,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850436 2024-11-14T14:00:50,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850436) roll requested 2024-11-14T14:00:50,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850442 2024-11-14T14:00:50,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850442 2024-11-14T14:00:50,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850442) roll requested 2024-11-14T14:00:50,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850449 2024-11-14T14:00:50,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850449 2024-11-14T14:00:50,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850449) roll requested 2024-11-14T14:00:50,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850474 2024-11-14T14:00:50,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850474 2024-11-14T14:00:50,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850474) roll requested 2024-11-14T14:00:50,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850481 2024-11-14T14:00:50,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850481 2024-11-14T14:00:50,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850481) roll requested 2024-11-14T14:00:50,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850490 2024-11-14T14:00:50,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850490 2024-11-14T14:00:50,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850490) roll requested 2024-11-14T14:00:50,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850499 2024-11-14T14:00:50,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850499 2024-11-14T14:00:50,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850499) roll requested 2024-11-14T14:00:50,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850506 2024-11-14T14:00:50,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850506 2024-11-14T14:00:50,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850506) roll requested 2024-11-14T14:00:50,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850513 2024-11-14T14:00:50,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850513 2024-11-14T14:00:50,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850513) roll requested 2024-11-14T14:00:50,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850525 2024-11-14T14:00:50,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850525 2024-11-14T14:00:50,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850525) roll requested 2024-11-14T14:00:50,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850532 2024-11-14T14:00:50,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850532 2024-11-14T14:00:50,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850532) roll requested 2024-11-14T14:00:50,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850540 2024-11-14T14:00:50,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850540 2024-11-14T14:00:50,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850540) roll requested 2024-11-14T14:00:50,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850547 2024-11-14T14:00:50,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850547 2024-11-14T14:00:50,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850547) roll requested 2024-11-14T14:00:50,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850555 2024-11-14T14:00:50,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850555 2024-11-14T14:00:50,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850555) roll requested 2024-11-14T14:00:50,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850562 2024-11-14T14:00:50,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850562 2024-11-14T14:00:50,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850562) roll requested 2024-11-14T14:00:50,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850569 2024-11-14T14:00:50,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850569 2024-11-14T14:00:50,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850569) roll requested 2024-11-14T14:00:50,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850581 2024-11-14T14:00:50,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850581 2024-11-14T14:00:50,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850581) roll requested 2024-11-14T14:00:50,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850589 2024-11-14T14:00:50,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850589 2024-11-14T14:00:50,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850589) roll requested 2024-11-14T14:00:50,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850596 2024-11-14T14:00:50,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850596 2024-11-14T14:00:50,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850596) roll requested 2024-11-14T14:00:50,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850603 2024-11-14T14:00:50,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850603 2024-11-14T14:00:50,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850603) roll requested 2024-11-14T14:00:50,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850613 2024-11-14T14:00:50,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850613 2024-11-14T14:00:50,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850613) roll requested 2024-11-14T14:00:50,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850619 2024-11-14T14:00:50,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850619 2024-11-14T14:00:50,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850619) roll requested 2024-11-14T14:00:50,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850627 2024-11-14T14:00:50,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850627 2024-11-14T14:00:50,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850627) roll requested 2024-11-14T14:00:50,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850637 2024-11-14T14:00:50,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850637 2024-11-14T14:00:50,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850637) roll requested 2024-11-14T14:00:50,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850644 2024-11-14T14:00:50,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850644 2024-11-14T14:00:50,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850644) roll requested 2024-11-14T14:00:50,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850651 2024-11-14T14:00:50,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850651 2024-11-14T14:00:50,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850651) roll requested 2024-11-14T14:00:50,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850661 2024-11-14T14:00:50,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850661 2024-11-14T14:00:50,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850661) roll requested 2024-11-14T14:00:50,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850668 2024-11-14T14:00:50,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850668 2024-11-14T14:00:50,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850668) roll requested 2024-11-14T14:00:50,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850677 2024-11-14T14:00:50,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850677 2024-11-14T14:00:50,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850677) roll requested 2024-11-14T14:00:50,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850684 2024-11-14T14:00:50,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850684 2024-11-14T14:00:50,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850684) roll requested 2024-11-14T14:00:50,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850693 2024-11-14T14:00:50,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850693 2024-11-14T14:00:50,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850693) roll requested 2024-11-14T14:00:50,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850701 2024-11-14T14:00:50,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850701 2024-11-14T14:00:50,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850701) roll requested 2024-11-14T14:00:50,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850708 2024-11-14T14:00:50,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850708 2024-11-14T14:00:50,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850708) roll requested 2024-11-14T14:00:50,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850715 2024-11-14T14:00:50,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850715 2024-11-14T14:00:50,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850715) roll requested 2024-11-14T14:00:50,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850722 2024-11-14T14:00:50,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850722 2024-11-14T14:00:50,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850722) roll requested 2024-11-14T14:00:50,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850729 2024-11-14T14:00:50,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850729 2024-11-14T14:00:50,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850729) roll requested 2024-11-14T14:00:50,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850736 2024-11-14T14:00:50,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850736 2024-11-14T14:00:50,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850736) roll requested 2024-11-14T14:00:50,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850743 2024-11-14T14:00:50,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850743 2024-11-14T14:00:50,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850743) roll requested 2024-11-14T14:00:50,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850750 2024-11-14T14:00:50,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850750 2024-11-14T14:00:50,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850750) roll requested 2024-11-14T14:00:50,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850757 2024-11-14T14:00:50,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850757 2024-11-14T14:00:50,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850757) roll requested 2024-11-14T14:00:50,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850764 2024-11-14T14:00:50,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850764 2024-11-14T14:00:50,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850764) roll requested 2024-11-14T14:00:50,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850770 2024-11-14T14:00:50,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850770 2024-11-14T14:00:50,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850770) roll requested 2024-11-14T14:00:50,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850777 2024-11-14T14:00:50,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850777 2024-11-14T14:00:50,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850777) roll requested 2024-11-14T14:00:50,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850784 2024-11-14T14:00:50,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850784 2024-11-14T14:00:50,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850784) roll requested 2024-11-14T14:00:50,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850791 2024-11-14T14:00:50,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850791 2024-11-14T14:00:50,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850791) roll requested 2024-11-14T14:00:50,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850799 2024-11-14T14:00:50,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850799 2024-11-14T14:00:50,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850799) roll requested 2024-11-14T14:00:50,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850806 2024-11-14T14:00:50,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850806 2024-11-14T14:00:50,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850806) roll requested 2024-11-14T14:00:50,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850812 2024-11-14T14:00:50,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850812 2024-11-14T14:00:50,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850812) roll requested 2024-11-14T14:00:50,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850819 2024-11-14T14:00:50,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850819 2024-11-14T14:00:50,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850819) roll requested 2024-11-14T14:00:50,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850826 2024-11-14T14:00:50,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850826 2024-11-14T14:00:50,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850826) roll requested 2024-11-14T14:00:50,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850833 2024-11-14T14:00:50,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850833 2024-11-14T14:00:50,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850833) roll requested 2024-11-14T14:00:50,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850840 2024-11-14T14:00:50,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850840 2024-11-14T14:00:50,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850840) roll requested 2024-11-14T14:00:50,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850846 2024-11-14T14:00:50,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850846 2024-11-14T14:00:50,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850846) roll requested 2024-11-14T14:00:50,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850853 2024-11-14T14:00:50,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850853 2024-11-14T14:00:50,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850853) roll requested 2024-11-14T14:00:50,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850860 2024-11-14T14:00:50,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850860 2024-11-14T14:00:50,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850860) roll requested 2024-11-14T14:00:50,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850867 2024-11-14T14:00:50,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850867 2024-11-14T14:00:50,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850867) roll requested 2024-11-14T14:00:50,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850874 2024-11-14T14:00:50,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850874 2024-11-14T14:00:50,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850874) roll requested 2024-11-14T14:00:50,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850881 2024-11-14T14:00:50,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850881 2024-11-14T14:00:50,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850881) roll requested 2024-11-14T14:00:50,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850888 2024-11-14T14:00:50,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850888 2024-11-14T14:00:50,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850888) roll requested 2024-11-14T14:00:50,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850894 2024-11-14T14:00:50,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850894 2024-11-14T14:00:50,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850894) roll requested 2024-11-14T14:00:50,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850901 2024-11-14T14:00:50,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850901 2024-11-14T14:00:50,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850901) roll requested 2024-11-14T14:00:50,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850908 2024-11-14T14:00:50,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850908 2024-11-14T14:00:50,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850908) roll requested 2024-11-14T14:00:50,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850915 2024-11-14T14:00:50,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850915 2024-11-14T14:00:50,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850915) roll requested 2024-11-14T14:00:50,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850922 2024-11-14T14:00:50,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850922 2024-11-14T14:00:50,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850922) roll requested 2024-11-14T14:00:50,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850929 2024-11-14T14:00:50,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850929 2024-11-14T14:00:50,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850929) roll requested 2024-11-14T14:00:50,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850936 2024-11-14T14:00:50,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850936 2024-11-14T14:00:50,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850936) roll requested 2024-11-14T14:00:50,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850942 2024-11-14T14:00:50,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850942 2024-11-14T14:00:50,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850942) roll requested 2024-11-14T14:00:50,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850949 2024-11-14T14:00:50,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850949 2024-11-14T14:00:50,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850949) roll requested 2024-11-14T14:00:50,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850956 2024-11-14T14:00:50,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850956 2024-11-14T14:00:50,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850956) roll requested 2024-11-14T14:00:50,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850963 2024-11-14T14:00:50,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850963 2024-11-14T14:00:50,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850963) roll requested 2024-11-14T14:00:50,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850970 2024-11-14T14:00:50,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850970 2024-11-14T14:00:50,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850970) roll requested 2024-11-14T14:00:50,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850977 2024-11-14T14:00:50,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850977 2024-11-14T14:00:50,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850977) roll requested 2024-11-14T14:00:50,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850984 2024-11-14T14:00:50,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850984 2024-11-14T14:00:50,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850984) roll requested 2024-11-14T14:00:50,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850991 2024-11-14T14:00:50,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:50,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850991 2024-11-14T14:00:50,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:50,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:50,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850991) roll requested 2024-11-14T14:00:50,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592850998 2024-11-14T14:00:51,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850998 2024-11-14T14:00:51,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:51,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:51,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592850998) roll requested 2024-11-14T14:00:51,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851004 2024-11-14T14:00:51,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592850998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851004 2024-11-14T14:00:51,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:51,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:51,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851004) roll requested 2024-11-14T14:00:51,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851012 2024-11-14T14:00:51,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851012 2024-11-14T14:00:51,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:51,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:51,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851012) roll requested 2024-11-14T14:00:51,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851018 2024-11-14T14:00:51,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851018 2024-11-14T14:00:51,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:51,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:51,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851018) roll requested 2024-11-14T14:00:51,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851025 2024-11-14T14:00:51,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851025 2024-11-14T14:00:51,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:51,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:51,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851025) roll requested 2024-11-14T14:00:51,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851032 2024-11-14T14:00:51,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851032 2024-11-14T14:00:51,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:51,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:51,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851032) roll requested 2024-11-14T14:00:51,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851038 2024-11-14T14:00:51,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851038 2024-11-14T14:00:51,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:51,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:51,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851038) roll requested 2024-11-14T14:00:51,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851053 2024-11-14T14:00:51,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851053 2024-11-14T14:00:51,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:51,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:00:51,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851053) roll requested 2024-11-14T14:00:51,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851065 2024-11-14T14:00:51,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851065 2024-11-14T14:00:51,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851065) roll requested 2024-11-14T14:00:51,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851076 2024-11-14T14:00:51,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851076 2024-11-14T14:00:51,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851076) roll requested 2024-11-14T14:00:51,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851086 2024-11-14T14:00:51,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851086 2024-11-14T14:00:51,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851086) roll requested 2024-11-14T14:00:51,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851097 2024-11-14T14:00:51,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851097 2024-11-14T14:00:51,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851097) roll requested 2024-11-14T14:00:51,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851106 2024-11-14T14:00:51,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851106 2024-11-14T14:00:51,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851106) roll requested 2024-11-14T14:00:51,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851114 2024-11-14T14:00:51,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851106 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851114 2024-11-14T14:00:51,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851114) roll requested 2024-11-14T14:00:51,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851125 2024-11-14T14:00:51,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851114 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851125 2024-11-14T14:00:51,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851125) roll requested 2024-11-14T14:00:51,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851133 2024-11-14T14:00:51,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851133 2024-11-14T14:00:51,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851133) roll requested 2024-11-14T14:00:51,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851141 2024-11-14T14:00:51,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851141 2024-11-14T14:00:51,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851141) roll requested 2024-11-14T14:00:51,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851149 2024-11-14T14:00:51,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851149 2024-11-14T14:00:51,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,157 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851149) roll requested 2024-11-14T14:00:51,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851157 2024-11-14T14:00:51,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851157 2024-11-14T14:00:51,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,164 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851157) roll requested 2024-11-14T14:00:51,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851165 2024-11-14T14:00:51,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851165 2024-11-14T14:00:51,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851165) roll requested 2024-11-14T14:00:51,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851172 2024-11-14T14:00:51,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851172 2024-11-14T14:00:51,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851172) roll requested 2024-11-14T14:00:51,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851180 2024-11-14T14:00:51,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851172 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851180 2024-11-14T14:00:51,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,186 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851180) roll requested 2024-11-14T14:00:51,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851186 2024-11-14T14:00:51,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851186 2024-11-14T14:00:51,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851186) roll requested 2024-11-14T14:00:51,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851194 2024-11-14T14:00:51,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851186 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851194 2024-11-14T14:00:51,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851194) roll requested 2024-11-14T14:00:51,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851201 2024-11-14T14:00:51,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851194 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851201 2024-11-14T14:00:51,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851201) roll requested 2024-11-14T14:00:51,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851210 2024-11-14T14:00:51,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851210 2024-11-14T14:00:51,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851210) roll requested 2024-11-14T14:00:51,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851217 2024-11-14T14:00:51,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851217 2024-11-14T14:00:51,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851217) roll requested 2024-11-14T14:00:51,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851226 2024-11-14T14:00:51,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851226 2024-11-14T14:00:51,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851226) roll requested 2024-11-14T14:00:51,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851234 2024-11-14T14:00:51,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851234 2024-11-14T14:00:51,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851234) roll requested 2024-11-14T14:00:51,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851242 2024-11-14T14:00:51,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851234 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851242 2024-11-14T14:00:51,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,250 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851242) roll requested 2024-11-14T14:00:51,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851250 2024-11-14T14:00:51,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851242 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851250 2024-11-14T14:00:51,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851250) roll requested 2024-11-14T14:00:51,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851257 2024-11-14T14:00:51,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851250 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851257 2024-11-14T14:00:51,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851257) roll requested 2024-11-14T14:00:51,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851264 2024-11-14T14:00:51,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851264 2024-11-14T14:00:51,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851264) roll requested 2024-11-14T14:00:51,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851271 2024-11-14T14:00:51,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851271 2024-11-14T14:00:51,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851271) roll requested 2024-11-14T14:00:51,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851278 2024-11-14T14:00:51,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851278 2024-11-14T14:00:51,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851278) roll requested 2024-11-14T14:00:51,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851285 2024-11-14T14:00:51,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851285 2024-11-14T14:00:51,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851285) roll requested 2024-11-14T14:00:51,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851292 2024-11-14T14:00:51,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851292 2024-11-14T14:00:51,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,299 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851292) roll requested 2024-11-14T14:00:51,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851299 2024-11-14T14:00:51,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851299 2024-11-14T14:00:51,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851299) roll requested 2024-11-14T14:00:51,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851306 2024-11-14T14:00:51,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851306 2024-11-14T14:00:51,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851306) roll requested 2024-11-14T14:00:51,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851314 2024-11-14T14:00:51,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851306 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851314 2024-11-14T14:00:51,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851314) roll requested 2024-11-14T14:00:51,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851321 2024-11-14T14:00:51,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851321 2024-11-14T14:00:51,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,330 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851321) roll requested 2024-11-14T14:00:51,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851330 2024-11-14T14:00:51,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851330 2024-11-14T14:00:51,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,338 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851330) roll requested 2024-11-14T14:00:51,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851339 2024-11-14T14:00:51,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851330 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851339 2024-11-14T14:00:51,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851339) roll requested 2024-11-14T14:00:51,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851348 2024-11-14T14:00:51,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851348 2024-11-14T14:00:51,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851348) roll requested 2024-11-14T14:00:51,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851358 2024-11-14T14:00:51,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851358 2024-11-14T14:00:51,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851358) roll requested 2024-11-14T14:00:51,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851387 2024-11-14T14:00:51,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851387 2024-11-14T14:00:51,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851387) roll requested 2024-11-14T14:00:51,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851397 2024-11-14T14:00:51,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851397 2024-11-14T14:00:51,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,409 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851397) roll requested 2024-11-14T14:00:51,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851409 2024-11-14T14:00:51,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851409 2024-11-14T14:00:51,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,421 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851409) roll requested 2024-11-14T14:00:51,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851421 2024-11-14T14:00:51,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851421 2024-11-14T14:00:51,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,433 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851421) roll requested 2024-11-14T14:00:51,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851433 2024-11-14T14:00:51,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851433 2024-11-14T14:00:51,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851433) roll requested 2024-11-14T14:00:51,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851441 2024-11-14T14:00:51,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851441 2024-11-14T14:00:51,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851441) roll requested 2024-11-14T14:00:51,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851451 2024-11-14T14:00:51,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851451 2024-11-14T14:00:51,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851451) roll requested 2024-11-14T14:00:51,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851458 2024-11-14T14:00:51,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851458 2024-11-14T14:00:51,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851458) roll requested 2024-11-14T14:00:51,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851469 2024-11-14T14:00:51,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851469 2024-11-14T14:00:51,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851469) roll requested 2024-11-14T14:00:51,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851481 2024-11-14T14:00:51,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851481 2024-11-14T14:00:51,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851481) roll requested 2024-11-14T14:00:51,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851497 2024-11-14T14:00:51,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851481 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851497 2024-11-14T14:00:51,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,513 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851497) roll requested 2024-11-14T14:00:51,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851513 2024-11-14T14:00:51,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851513 2024-11-14T14:00:51,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851513) roll requested 2024-11-14T14:00:51,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851525 2024-11-14T14:00:51,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851525 2024-11-14T14:00:51,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,532 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851525) roll requested 2024-11-14T14:00:51,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851532 2024-11-14T14:00:51,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851532 2024-11-14T14:00:51,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,539 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851532) roll requested 2024-11-14T14:00:51,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851539 2024-11-14T14:00:51,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851539 2024-11-14T14:00:51,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851539) roll requested 2024-11-14T14:00:51,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851548 2024-11-14T14:00:51,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851539 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851548 2024-11-14T14:00:51,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851548) roll requested 2024-11-14T14:00:51,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851555 2024-11-14T14:00:51,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851555 2024-11-14T14:00:51,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851555) roll requested 2024-11-14T14:00:51,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851563 2024-11-14T14:00:51,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851563 2024-11-14T14:00:51,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851563) roll requested 2024-11-14T14:00:51,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851570 2024-11-14T14:00:51,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851570 2024-11-14T14:00:51,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,578 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=56, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,579 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851570) roll requested 2024-11-14T14:00:51,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851579 2024-11-14T14:00:51,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851570 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851579 2024-11-14T14:00:51,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=57, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,586 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851579) roll requested 2024-11-14T14:00:51,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851587 2024-11-14T14:00:51,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851587 2024-11-14T14:00:51,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=58, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851587) roll requested 2024-11-14T14:00:51,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851594 2024-11-14T14:00:51,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851594 2024-11-14T14:00:51,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=59, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851594) roll requested 2024-11-14T14:00:51,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851601 2024-11-14T14:00:51,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851594 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851601 2024-11-14T14:00:51,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=60, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851601) roll requested 2024-11-14T14:00:51,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851608 2024-11-14T14:00:51,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851608 2024-11-14T14:00:51,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=61, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,615 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851608) roll requested 2024-11-14T14:00:51,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851615 2024-11-14T14:00:51,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851615 2024-11-14T14:00:51,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=62, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,622 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851615) roll requested 2024-11-14T14:00:51,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851622 2024-11-14T14:00:51,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851615 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851622 2024-11-14T14:00:51,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=63, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851622) roll requested 2024-11-14T14:00:51,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851629 2024-11-14T14:00:51,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851622 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851629 2024-11-14T14:00:51,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,636 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=64, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,636 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,636 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851629) roll requested 2024-11-14T14:00:51,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851636 2024-11-14T14:00:51,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851636 2024-11-14T14:00:51,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,643 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=65, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,643 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,643 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851636) roll requested 2024-11-14T14:00:51,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851643 2024-11-14T14:00:51,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851636 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851643 2024-11-14T14:00:51,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,650 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=66, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,650 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,650 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851643) roll requested 2024-11-14T14:00:51,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851650 2024-11-14T14:00:51,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851643 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851650 2024-11-14T14:00:51,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=67, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851650) roll requested 2024-11-14T14:00:51,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851658 2024-11-14T14:00:51,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851650 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851658 2024-11-14T14:00:51,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=68, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851658) roll requested 2024-11-14T14:00:51,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851665 2024-11-14T14:00:51,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851665 2024-11-14T14:00:51,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,675 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=69, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851665) roll requested 2024-11-14T14:00:51,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851676 2024-11-14T14:00:51,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851676 2024-11-14T14:00:51,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,685 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=70, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,685 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,685 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851676) roll requested 2024-11-14T14:00:51,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851685 2024-11-14T14:00:51,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851685 2024-11-14T14:00:51,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=71, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851685) roll requested 2024-11-14T14:00:51,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851693 2024-11-14T14:00:51,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851693 2024-11-14T14:00:51,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=72, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851693) roll requested 2024-11-14T14:00:51,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851700 2024-11-14T14:00:51,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851700 2024-11-14T14:00:51,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,707 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=73, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,707 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851700) roll requested 2024-11-14T14:00:51,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851707 2024-11-14T14:00:51,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851700 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851707 2024-11-14T14:00:51,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=74, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851707) roll requested 2024-11-14T14:00:51,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851717 2024-11-14T14:00:51,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851717 2024-11-14T14:00:51,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=75, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,728 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851717) roll requested 2024-11-14T14:00:51,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851728 2024-11-14T14:00:51,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851728 2024-11-14T14:00:51,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,738 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=76, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,738 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,738 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851728) roll requested 2024-11-14T14:00:51,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851738 2024-11-14T14:00:51,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851738 2024-11-14T14:00:51,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=77, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851738) roll requested 2024-11-14T14:00:51,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851749 2024-11-14T14:00:51,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851738 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851749 2024-11-14T14:00:51,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=78, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851749) roll requested 2024-11-14T14:00:51,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851756 2024-11-14T14:00:51,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851756 2024-11-14T14:00:51,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,763 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=79, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851756) roll requested 2024-11-14T14:00:51,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851764 2024-11-14T14:00:51,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851764 2024-11-14T14:00:51,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,770 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=80, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,770 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,770 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851764) roll requested 2024-11-14T14:00:51,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851770 2024-11-14T14:00:51,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851764 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851770 2024-11-14T14:00:51,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=81, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851770) roll requested 2024-11-14T14:00:51,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851777 2024-11-14T14:00:51,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851770 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851777 2024-11-14T14:00:51,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=82, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851777) roll requested 2024-11-14T14:00:51,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851785 2024-11-14T14:00:51,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851785 2024-11-14T14:00:51,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=83, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851785) roll requested 2024-11-14T14:00:51,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851793 2024-11-14T14:00:51,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851793 2024-11-14T14:00:51,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,803 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=84, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,803 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851793) roll requested 2024-11-14T14:00:51,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851803 2024-11-14T14:00:51,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851803 2024-11-14T14:00:51,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=85, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,810 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,810 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851803) roll requested 2024-11-14T14:00:51,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851810 2024-11-14T14:00:51,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851810 2024-11-14T14:00:51,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=86, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,817 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851810) roll requested 2024-11-14T14:00:51,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851817 2024-11-14T14:00:51,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851810 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851817 2024-11-14T14:00:51,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=87, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851817) roll requested 2024-11-14T14:00:51,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851825 2024-11-14T14:00:51,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851825 2024-11-14T14:00:51,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=88, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,832 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851825) roll requested 2024-11-14T14:00:51,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851832 2024-11-14T14:00:51,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851832 2024-11-14T14:00:51,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=89, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851832) roll requested 2024-11-14T14:00:51,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851839 2024-11-14T14:00:51,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851832 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851839 2024-11-14T14:00:51,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=90, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851839) roll requested 2024-11-14T14:00:51,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851846 2024-11-14T14:00:51,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851846 2024-11-14T14:00:51,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=91, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851846) roll requested 2024-11-14T14:00:51,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851853 2024-11-14T14:00:51,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851853 2024-11-14T14:00:51,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=92, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851853) roll requested 2024-11-14T14:00:51,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851860 2024-11-14T14:00:51,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851860 2024-11-14T14:00:51,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,867 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=93, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,867 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,867 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851860) roll requested 2024-11-14T14:00:51,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851868 2024-11-14T14:00:51,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851868 2024-11-14T14:00:51,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=94, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851868) roll requested 2024-11-14T14:00:51,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851874 2024-11-14T14:00:51,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851868 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851874 2024-11-14T14:00:51,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=95, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,882 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,883 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851874) roll requested 2024-11-14T14:00:51,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851883 2024-11-14T14:00:51,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851883 2024-11-14T14:00:51,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,889 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=96, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,889 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851883) roll requested 2024-11-14T14:00:51,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851890 2024-11-14T14:00:51,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851890 2024-11-14T14:00:51,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,896 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=97, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,896 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851890) roll requested 2024-11-14T14:00:51,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851897 2024-11-14T14:00:51,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851897 2024-11-14T14:00:51,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,903 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=98, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,903 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851897) roll requested 2024-11-14T14:00:51,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851904 2024-11-14T14:00:51,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851904 2024-11-14T14:00:51,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,910 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=99, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,910 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851904) roll requested 2024-11-14T14:00:51,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851910 2024-11-14T14:00:51,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851904 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851910 2024-11-14T14:00:51,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=100, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851910) roll requested 2024-11-14T14:00:51,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851917 2024-11-14T14:00:51,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851917 2024-11-14T14:00:51,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,923 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=101, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,923 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851917) roll requested 2024-11-14T14:00:51,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851924 2024-11-14T14:00:51,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851924 2024-11-14T14:00:51,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=102, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851924) roll requested 2024-11-14T14:00:51,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851931 2024-11-14T14:00:51,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851931 2024-11-14T14:00:51,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=103, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851931) roll requested 2024-11-14T14:00:51,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851938 2024-11-14T14:00:51,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851938 2024-11-14T14:00:51,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=104, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851938) roll requested 2024-11-14T14:00:51,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851945 2024-11-14T14:00:51,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851945 2024-11-14T14:00:51,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=105, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851945) roll requested 2024-11-14T14:00:51,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851953 2024-11-14T14:00:51,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851953 2024-11-14T14:00:51,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=106, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851953) roll requested 2024-11-14T14:00:51,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851960 2024-11-14T14:00:51,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851960 2024-11-14T14:00:51,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=107, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851960) roll requested 2024-11-14T14:00:51,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851967 2024-11-14T14:00:51,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851967 2024-11-14T14:00:51,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=108, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851967) roll requested 2024-11-14T14:00:51,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851973 2024-11-14T14:00:51,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851973 2024-11-14T14:00:51,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,980 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=109, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,980 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,980 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851973) roll requested 2024-11-14T14:00:51,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851980 2024-11-14T14:00:51,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:51,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851980 2024-11-14T14:00:51,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:51,990 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=110, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:51,990 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:51,990 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:51,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851980) roll requested 2024-11-14T14:00:51,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592851990 2024-11-14T14:00:52,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851980 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851990 2024-11-14T14:00:52,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,010 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=111, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592851990) roll requested 2024-11-14T14:00:52,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852011 2024-11-14T14:00:52,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592851990 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852011 2024-11-14T14:00:52,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=112, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852011) roll requested 2024-11-14T14:00:52,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852025 2024-11-14T14:00:52,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852025 2024-11-14T14:00:52,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,040 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=113, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852025) roll requested 2024-11-14T14:00:52,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852041 2024-11-14T14:00:52,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852041 2024-11-14T14:00:52,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=114, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,053 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852041) roll requested 2024-11-14T14:00:52,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852053 2024-11-14T14:00:52,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852053 2024-11-14T14:00:52,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,065 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=115, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852053) roll requested 2024-11-14T14:00:52,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852065 2024-11-14T14:00:52,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852065 2024-11-14T14:00:52,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=116, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852065) roll requested 2024-11-14T14:00:52,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852073 2024-11-14T14:00:52,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852073 2024-11-14T14:00:52,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,087 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=117, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,087 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,088 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852073) roll requested 2024-11-14T14:00:52,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852088 2024-11-14T14:00:52,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852088 2024-11-14T14:00:52,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,095 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=118, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,095 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,095 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852088) roll requested 2024-11-14T14:00:52,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852095 2024-11-14T14:00:52,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852088 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852095 2024-11-14T14:00:52,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=119, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852095) roll requested 2024-11-14T14:00:52,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852105 2024-11-14T14:00:52,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852095 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852105 2024-11-14T14:00:52,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=120, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852105) roll requested 2024-11-14T14:00:52,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852113 2024-11-14T14:00:52,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852113 2024-11-14T14:00:52,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=121, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852113) roll requested 2024-11-14T14:00:52,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852121 2024-11-14T14:00:52,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852121 2024-11-14T14:00:52,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=122, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852121) roll requested 2024-11-14T14:00:52,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852129 2024-11-14T14:00:52,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852129 2024-11-14T14:00:52,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=123, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852129) roll requested 2024-11-14T14:00:52,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852136 2024-11-14T14:00:52,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852136 2024-11-14T14:00:52,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,143 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=124, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,143 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852136) roll requested 2024-11-14T14:00:52,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852143 2024-11-14T14:00:52,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852143 2024-11-14T14:00:52,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=125, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,150 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852143) roll requested 2024-11-14T14:00:52,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852150 2024-11-14T14:00:52,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852143 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852150 2024-11-14T14:00:52,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=126, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852150) roll requested 2024-11-14T14:00:52,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852161 2024-11-14T14:00:52,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852150 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852161 2024-11-14T14:00:52,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=127, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,169 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,169 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852161) roll requested 2024-11-14T14:00:52,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852169 2024-11-14T14:00:52,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852169 2024-11-14T14:00:52,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=128, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852169) roll requested 2024-11-14T14:00:52,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852177 2024-11-14T14:00:52,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852169 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852177 2024-11-14T14:00:52,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,187 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=129, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,187 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,187 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852177) roll requested 2024-11-14T14:00:52,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852187 2024-11-14T14:00:52,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852187 2024-11-14T14:00:52,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=130, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852187) roll requested 2024-11-14T14:00:52,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852195 2024-11-14T14:00:52,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852187 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852195 2024-11-14T14:00:52,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=131, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,202 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852195) roll requested 2024-11-14T14:00:52,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852202 2024-11-14T14:00:52,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852202 2024-11-14T14:00:52,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=132, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852202) roll requested 2024-11-14T14:00:52,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852209 2024-11-14T14:00:52,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852202 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852209 2024-11-14T14:00:52,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=133, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852209) roll requested 2024-11-14T14:00:52,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852217 2024-11-14T14:00:52,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852217 2024-11-14T14:00:52,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=134, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852217) roll requested 2024-11-14T14:00:52,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852224 2024-11-14T14:00:52,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852224 2024-11-14T14:00:52,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=135, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,231 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852224) roll requested 2024-11-14T14:00:52,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852231 2024-11-14T14:00:52,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852224 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852231 2024-11-14T14:00:52,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=136, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852231) roll requested 2024-11-14T14:00:52,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852237 2024-11-14T14:00:52,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852231 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852237 2024-11-14T14:00:52,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=137, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852237) roll requested 2024-11-14T14:00:52,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852245 2024-11-14T14:00:52,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852245 2024-11-14T14:00:52,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=138, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852245) roll requested 2024-11-14T14:00:52,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852261 2024-11-14T14:00:52,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852261 2024-11-14T14:00:52,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=139, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852261) roll requested 2024-11-14T14:00:52,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852268 2024-11-14T14:00:52,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852268 2024-11-14T14:00:52,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,275 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=140, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,275 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,275 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852268) roll requested 2024-11-14T14:00:52,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852275 2024-11-14T14:00:52,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852275 2024-11-14T14:00:52,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=141, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852275) roll requested 2024-11-14T14:00:52,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852283 2024-11-14T14:00:52,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852275 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852283 2024-11-14T14:00:52,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=142, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,290 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,290 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852283) roll requested 2024-11-14T14:00:52,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852290 2024-11-14T14:00:52,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852290 2024-11-14T14:00:52,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=143, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852290) roll requested 2024-11-14T14:00:52,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852297 2024-11-14T14:00:52,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852297 2024-11-14T14:00:52,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=144, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852297) roll requested 2024-11-14T14:00:52,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852304 2024-11-14T14:00:52,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852304 2024-11-14T14:00:52,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=145, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852304) roll requested 2024-11-14T14:00:52,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852311 2024-11-14T14:00:52,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852311 2024-11-14T14:00:52,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=146, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852311) roll requested 2024-11-14T14:00:52,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852318 2024-11-14T14:00:52,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852318 2024-11-14T14:00:52,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,324 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=147, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852318) roll requested 2024-11-14T14:00:52,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852324 2024-11-14T14:00:52,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852324 2024-11-14T14:00:52,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,331 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=148, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,331 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,331 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852324) roll requested 2024-11-14T14:00:52,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852331 2024-11-14T14:00:52,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852331 2024-11-14T14:00:52,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,338 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=149, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,338 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,338 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852331) roll requested 2024-11-14T14:00:52,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852338 2024-11-14T14:00:52,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852338 2024-11-14T14:00:52,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,352 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=150, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,353 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852338) roll requested 2024-11-14T14:00:52,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852353 2024-11-14T14:00:52,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852338 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852353 2024-11-14T14:00:52,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,359 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=151, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,359 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,360 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852353) roll requested 2024-11-14T14:00:52,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852360 2024-11-14T14:00:52,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852353 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852360 2024-11-14T14:00:52,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,366 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=152, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,366 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,367 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852360) roll requested 2024-11-14T14:00:52,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852367 2024-11-14T14:00:52,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852367 2024-11-14T14:00:52,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=153, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,373 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852367) roll requested 2024-11-14T14:00:52,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852374 2024-11-14T14:00:52,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852367 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852374 2024-11-14T14:00:52,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=154, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852374) roll requested 2024-11-14T14:00:52,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852380 2024-11-14T14:00:52,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852380 2024-11-14T14:00:52,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=155, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852380) roll requested 2024-11-14T14:00:52,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852388 2024-11-14T14:00:52,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852388 2024-11-14T14:00:52,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=156, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852388) roll requested 2024-11-14T14:00:52,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852394 2024-11-14T14:00:52,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852388 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852394 2024-11-14T14:00:52,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=157, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852394) roll requested 2024-11-14T14:00:52,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852402 2024-11-14T14:00:52,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852402 2024-11-14T14:00:52,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=158, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852402) roll requested 2024-11-14T14:00:52,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852409 2024-11-14T14:00:52,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852409 2024-11-14T14:00:52,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=159, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852409) roll requested 2024-11-14T14:00:52,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852416 2024-11-14T14:00:52,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852416 2024-11-14T14:00:52,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=160, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852416) roll requested 2024-11-14T14:00:52,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852424 2024-11-14T14:00:52,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852424 2024-11-14T14:00:52,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=161, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852424) roll requested 2024-11-14T14:00:52,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852435 2024-11-14T14:00:52,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852435 2024-11-14T14:00:52,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=162, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852435) roll requested 2024-11-14T14:00:52,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852443 2024-11-14T14:00:52,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852443 2024-11-14T14:00:52,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=163, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852443) roll requested 2024-11-14T14:00:52,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852450 2024-11-14T14:00:52,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852450 2024-11-14T14:00:52,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=164, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852450) roll requested 2024-11-14T14:00:52,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852457 2024-11-14T14:00:52,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852457 2024-11-14T14:00:52,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=165, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852457) roll requested 2024-11-14T14:00:52,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852464 2024-11-14T14:00:52,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852464 2024-11-14T14:00:52,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=166, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852464) roll requested 2024-11-14T14:00:52,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852471 2024-11-14T14:00:52,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852471 2024-11-14T14:00:52,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=167, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852471) roll requested 2024-11-14T14:00:52,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852478 2024-11-14T14:00:52,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852471 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852478 2024-11-14T14:00:52,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=168, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852478) roll requested 2024-11-14T14:00:52,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852485 2024-11-14T14:00:52,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852485 2024-11-14T14:00:52,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,494 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=169, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,494 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,494 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852485) roll requested 2024-11-14T14:00:52,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852494 2024-11-14T14:00:52,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852494 2024-11-14T14:00:52,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=170, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852494) roll requested 2024-11-14T14:00:52,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852501 2024-11-14T14:00:52,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852494 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852501 2024-11-14T14:00:52,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=171, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852501) roll requested 2024-11-14T14:00:52,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852508 2024-11-14T14:00:52,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852508 2024-11-14T14:00:52,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,515 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=172, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,515 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852508) roll requested 2024-11-14T14:00:52,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852515 2024-11-14T14:00:52,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852515 2024-11-14T14:00:52,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,522 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=173, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852515) roll requested 2024-11-14T14:00:52,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852522 2024-11-14T14:00:52,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852522 2024-11-14T14:00:52,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=174, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852522) roll requested 2024-11-14T14:00:52,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852529 2024-11-14T14:00:52,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852529 2024-11-14T14:00:52,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,536 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=175, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,536 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852529) roll requested 2024-11-14T14:00:52,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852536 2024-11-14T14:00:52,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852536 2024-11-14T14:00:52,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=176, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852536) roll requested 2024-11-14T14:00:52,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852543 2024-11-14T14:00:52,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852543 2024-11-14T14:00:52,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=177, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852543) roll requested 2024-11-14T14:00:52,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852550 2024-11-14T14:00:52,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852550 2024-11-14T14:00:52,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,558 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=178, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852550) roll requested 2024-11-14T14:00:52,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852558 2024-11-14T14:00:52,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852558 2024-11-14T14:00:52,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,565 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=179, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,565 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852558) roll requested 2024-11-14T14:00:52,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852565 2024-11-14T14:00:52,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852565 2024-11-14T14:00:52,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=180, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852565) roll requested 2024-11-14T14:00:52,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852572 2024-11-14T14:00:52,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852572 2024-11-14T14:00:52,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,579 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=181, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,579 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852572) roll requested 2024-11-14T14:00:52,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852579 2024-11-14T14:00:52,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852579 2024-11-14T14:00:52,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=182, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,586 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852579) roll requested 2024-11-14T14:00:52,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852586 2024-11-14T14:00:52,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852586 2024-11-14T14:00:52,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=183, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852586) roll requested 2024-11-14T14:00:52,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852593 2024-11-14T14:00:52,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852586 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852593 2024-11-14T14:00:52,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=184, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852593) roll requested 2024-11-14T14:00:52,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852601 2024-11-14T14:00:52,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852601 2024-11-14T14:00:52,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=185, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852601) roll requested 2024-11-14T14:00:52,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852607 2024-11-14T14:00:52,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852607 2024-11-14T14:00:52,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=186, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852607) roll requested 2024-11-14T14:00:52,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852616 2024-11-14T14:00:52,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852616 2024-11-14T14:00:52,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,623 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=187, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,623 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852616) roll requested 2024-11-14T14:00:52,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852623 2024-11-14T14:00:52,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852623 2024-11-14T14:00:52,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=188, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852623) roll requested 2024-11-14T14:00:52,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852629 2024-11-14T14:00:52,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852629 2024-11-14T14:00:52,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,636 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=189, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,636 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,636 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852629) roll requested 2024-11-14T14:00:52,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852636 2024-11-14T14:00:52,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852636 2024-11-14T14:00:52,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=190, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852636) roll requested 2024-11-14T14:00:52,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852642 2024-11-14T14:00:52,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852636 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852642 2024-11-14T14:00:52,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,649 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=191, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,649 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,649 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852642) roll requested 2024-11-14T14:00:52,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852649 2024-11-14T14:00:52,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852649 2024-11-14T14:00:52,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,656 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=192, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,656 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852649) roll requested 2024-11-14T14:00:52,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852656 2024-11-14T14:00:52,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852656 2024-11-14T14:00:52,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=193, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852656) roll requested 2024-11-14T14:00:52,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852664 2024-11-14T14:00:52,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852664 2024-11-14T14:00:52,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,670 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=194, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852664) roll requested 2024-11-14T14:00:52,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852670 2024-11-14T14:00:52,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852670 2024-11-14T14:00:52,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,677 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=195, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,677 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,677 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852670) roll requested 2024-11-14T14:00:52,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852677 2024-11-14T14:00:52,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852677 2024-11-14T14:00:52,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=196, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852677) roll requested 2024-11-14T14:00:52,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852684 2024-11-14T14:00:52,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852677 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852684 2024-11-14T14:00:52,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,690 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=197, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,690 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,690 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852684) roll requested 2024-11-14T14:00:52,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852690 2024-11-14T14:00:52,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852690 2024-11-14T14:00:52,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=198, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852690) roll requested 2024-11-14T14:00:52,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852697 2024-11-14T14:00:52,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852690 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852697 2024-11-14T14:00:52,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=199, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,703 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852697) roll requested 2024-11-14T14:00:52,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852703 2024-11-14T14:00:52,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852703 2024-11-14T14:00:52,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,710 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=200, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,710 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,710 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852703) roll requested 2024-11-14T14:00:52,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852710 2024-11-14T14:00:52,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852710 2024-11-14T14:00:52,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=201, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852710) roll requested 2024-11-14T14:00:52,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852717 2024-11-14T14:00:52,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852710 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852717 2024-11-14T14:00:52,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=202, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852717) roll requested 2024-11-14T14:00:52,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852724 2024-11-14T14:00:52,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852724 2024-11-14T14:00:52,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,731 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=203, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,731 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,731 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852724) roll requested 2024-11-14T14:00:52,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852731 2024-11-14T14:00:52,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852731 2024-11-14T14:00:52,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,738 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=204, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,738 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,738 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852731) roll requested 2024-11-14T14:00:52,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852738 2024-11-14T14:00:52,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852731 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852738 2024-11-14T14:00:52,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=205, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852738) roll requested 2024-11-14T14:00:52,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852745 2024-11-14T14:00:52,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852738 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852745 2024-11-14T14:00:52,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=206, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852745) roll requested 2024-11-14T14:00:52,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852752 2024-11-14T14:00:52,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852752 2024-11-14T14:00:52,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=207, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852752) roll requested 2024-11-14T14:00:52,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852759 2024-11-14T14:00:52,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852759 2024-11-14T14:00:52,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=208, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852759) roll requested 2024-11-14T14:00:52,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852766 2024-11-14T14:00:52,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852766 2024-11-14T14:00:52,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=209, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852766) roll requested 2024-11-14T14:00:52,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852773 2024-11-14T14:00:52,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852773 2024-11-14T14:00:52,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,780 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=210, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,780 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,780 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852773) roll requested 2024-11-14T14:00:52,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852780 2024-11-14T14:00:52,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852780 2024-11-14T14:00:52,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,787 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=211, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,787 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852780) roll requested 2024-11-14T14:00:52,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852787 2024-11-14T14:00:52,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852780 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852787 2024-11-14T14:00:52,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=212, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,794 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852787) roll requested 2024-11-14T14:00:52,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852794 2024-11-14T14:00:52,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852787 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852794 2024-11-14T14:00:52,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=213, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852794) roll requested 2024-11-14T14:00:52,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852802 2024-11-14T14:00:52,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852802 2024-11-14T14:00:52,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,811 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=214, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852802) roll requested 2024-11-14T14:00:52,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852812 2024-11-14T14:00:52,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852812 2024-11-14T14:00:52,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=215, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852812) roll requested 2024-11-14T14:00:52,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852820 2024-11-14T14:00:52,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852820 2024-11-14T14:00:52,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=216, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852820) roll requested 2024-11-14T14:00:52,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852828 2024-11-14T14:00:52,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852828 2024-11-14T14:00:52,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,834 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,834 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=217, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,834 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852828) roll requested 2024-11-14T14:00:52,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852834 2024-11-14T14:00:52,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852834 2024-11-14T14:00:52,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,841 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=218, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852834) roll requested 2024-11-14T14:00:52,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852841 2024-11-14T14:00:52,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852834 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852841 2024-11-14T14:00:52,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=219, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852841) roll requested 2024-11-14T14:00:52,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852849 2024-11-14T14:00:52,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852849 2024-11-14T14:00:52,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=220, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852849) roll requested 2024-11-14T14:00:52,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852856 2024-11-14T14:00:52,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852856 2024-11-14T14:00:52,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,862 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,862 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=221, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,862 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852856) roll requested 2024-11-14T14:00:52,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852862 2024-11-14T14:00:52,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852862 2024-11-14T14:00:52,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,869 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,869 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=222, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,869 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852862) roll requested 2024-11-14T14:00:52,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852870 2024-11-14T14:00:52,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852862 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852870 2024-11-14T14:00:52,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,876 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=223, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,876 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,876 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852870) roll requested 2024-11-14T14:00:52,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852876 2024-11-14T14:00:52,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852876 2024-11-14T14:00:52,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,883 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=224, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,883 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852876) roll requested 2024-11-14T14:00:52,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852883 2024-11-14T14:00:52,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852883 2024-11-14T14:00:52,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=225, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852883) roll requested 2024-11-14T14:00:52,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852890 2024-11-14T14:00:52,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852890 2024-11-14T14:00:52,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=226, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852890) roll requested 2024-11-14T14:00:52,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852897 2024-11-14T14:00:52,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852897 2024-11-14T14:00:52,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,904 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=227, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,904 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,904 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852897) roll requested 2024-11-14T14:00:52,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852904 2024-11-14T14:00:52,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852904 2024-11-14T14:00:52,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,910 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=228, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,910 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852904) roll requested 2024-11-14T14:00:52,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852910 2024-11-14T14:00:52,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852904 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852910 2024-11-14T14:00:52,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=229, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,918 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852910) roll requested 2024-11-14T14:00:52,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852918 2024-11-14T14:00:52,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852918 2024-11-14T14:00:52,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=230, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852918) roll requested 2024-11-14T14:00:52,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852924 2024-11-14T14:00:52,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852924 2024-11-14T14:00:52,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=231, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852924) roll requested 2024-11-14T14:00:52,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852931 2024-11-14T14:00:52,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852931 2024-11-14T14:00:52,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=232, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852931) roll requested 2024-11-14T14:00:52,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852938 2024-11-14T14:00:52,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852938 2024-11-14T14:00:52,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=233, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852938) roll requested 2024-11-14T14:00:52,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852945 2024-11-14T14:00:52,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852945 2024-11-14T14:00:52,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=234, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852945) roll requested 2024-11-14T14:00:52,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852952 2024-11-14T14:00:52,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852952 2024-11-14T14:00:52,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=235, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852952) roll requested 2024-11-14T14:00:52,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852959 2024-11-14T14:00:52,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852959 2024-11-14T14:00:52,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=236, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852959) roll requested 2024-11-14T14:00:52,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852966 2024-11-14T14:00:52,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852966 2024-11-14T14:00:52,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=237, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852966) roll requested 2024-11-14T14:00:52,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852972 2024-11-14T14:00:52,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852972 2024-11-14T14:00:52,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=238, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852972) roll requested 2024-11-14T14:00:52,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852979 2024-11-14T14:00:52,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852979 2024-11-14T14:00:52,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=239, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852979) roll requested 2024-11-14T14:00:52,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852986 2024-11-14T14:00:52,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852986 2024-11-14T14:00:52,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:52,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=240, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:52,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:52,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:52,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852986) roll requested 2024-11-14T14:00:52,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592852993 2024-11-14T14:00:52,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:52,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852993 2024-11-14T14:00:53,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,000 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=241, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592852993) roll requested 2024-11-14T14:00:53,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853000 2024-11-14T14:00:53,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592852993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853000 2024-11-14T14:00:53,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=242, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853000) roll requested 2024-11-14T14:00:53,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853007 2024-11-14T14:00:53,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853007 2024-11-14T14:00:53,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,015 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,015 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=243, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,015 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853007) roll requested 2024-11-14T14:00:53,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853015 2024-11-14T14:00:53,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853015 2024-11-14T14:00:53,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=244, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853015) roll requested 2024-11-14T14:00:53,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853021 2024-11-14T14:00:53,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853021 2024-11-14T14:00:53,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=245, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853021) roll requested 2024-11-14T14:00:53,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853028 2024-11-14T14:00:53,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853028 2024-11-14T14:00:53,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,035 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=246, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,035 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,035 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853028) roll requested 2024-11-14T14:00:53,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853035 2024-11-14T14:00:53,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853035 2024-11-14T14:00:53,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,042 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=247, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853035) roll requested 2024-11-14T14:00:53,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853042 2024-11-14T14:00:53,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853042 2024-11-14T14:00:53,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,049 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=248, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853042) roll requested 2024-11-14T14:00:53,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853049 2024-11-14T14:00:53,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853042 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853049 2024-11-14T14:00:53,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=249, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853049) roll requested 2024-11-14T14:00:53,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853056 2024-11-14T14:00:53,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853056 2024-11-14T14:00:53,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=250, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853056) roll requested 2024-11-14T14:00:53,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853063 2024-11-14T14:00:53,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853056 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853063 2024-11-14T14:00:53,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=251, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853063) roll requested 2024-11-14T14:00:53,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853070 2024-11-14T14:00:53,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853070 2024-11-14T14:00:53,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=252, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853070) roll requested 2024-11-14T14:00:53,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853077 2024-11-14T14:00:53,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853077 2024-11-14T14:00:53,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=253, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853077) roll requested 2024-11-14T14:00:53,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853084 2024-11-14T14:00:53,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853084 2024-11-14T14:00:53,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=254, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853084) roll requested 2024-11-14T14:00:53,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853091 2024-11-14T14:00:53,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853091 2024-11-14T14:00:53,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=255, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853091) roll requested 2024-11-14T14:00:53,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853098 2024-11-14T14:00:53,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853098 2024-11-14T14:00:53,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=256, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853098) roll requested 2024-11-14T14:00:53,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853109 2024-11-14T14:00:53,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853109 2024-11-14T14:00:53,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=257, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853109) roll requested 2024-11-14T14:00:53,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853125 2024-11-14T14:00:53,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853125 2024-11-14T14:00:53,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=258, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853125) roll requested 2024-11-14T14:00:53,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853133 2024-11-14T14:00:53,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853133 2024-11-14T14:00:53,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=259, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853133) roll requested 2024-11-14T14:00:53,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853141 2024-11-14T14:00:53,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853141 2024-11-14T14:00:53,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=260, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853141) roll requested 2024-11-14T14:00:53,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853149 2024-11-14T14:00:53,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853149 2024-11-14T14:00:53,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=261, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853149) roll requested 2024-11-14T14:00:53,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853156 2024-11-14T14:00:53,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853156 2024-11-14T14:00:53,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,163 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,163 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=262, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,163 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853156) roll requested 2024-11-14T14:00:53,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853164 2024-11-14T14:00:53,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853164 2024-11-14T14:00:53,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,171 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,171 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=263, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,171 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853164) roll requested 2024-11-14T14:00:53,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853171 2024-11-14T14:00:53,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853164 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853171 2024-11-14T14:00:53,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=264, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853171) roll requested 2024-11-14T14:00:53,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853178 2024-11-14T14:00:53,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853178 2024-11-14T14:00:53,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=265, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853178) roll requested 2024-11-14T14:00:53,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853185 2024-11-14T14:00:53,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853185 2024-11-14T14:00:53,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,192 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=266, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853185) roll requested 2024-11-14T14:00:53,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853193 2024-11-14T14:00:53,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853193 2024-11-14T14:00:53,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=267, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853193) roll requested 2024-11-14T14:00:53,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853200 2024-11-14T14:00:53,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853193 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853200 2024-11-14T14:00:53,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=268, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,207 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853200) roll requested 2024-11-14T14:00:53,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853207 2024-11-14T14:00:53,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853207 2024-11-14T14:00:53,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,215 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=269, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,215 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,215 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853207) roll requested 2024-11-14T14:00:53,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853215 2024-11-14T14:00:53,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853207 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853215 2024-11-14T14:00:53,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,222 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=270, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853215) roll requested 2024-11-14T14:00:53,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853222 2024-11-14T14:00:53,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853215 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853222 2024-11-14T14:00:53,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=271, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853222) roll requested 2024-11-14T14:00:53,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853229 2024-11-14T14:00:53,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853222 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853229 2024-11-14T14:00:53,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=272, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853229) roll requested 2024-11-14T14:00:53,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853236 2024-11-14T14:00:53,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853236 2024-11-14T14:00:53,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,243 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=273, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853236) roll requested 2024-11-14T14:00:53,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853243 2024-11-14T14:00:53,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853243 2024-11-14T14:00:53,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,250 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,250 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=274, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,251 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853243) roll requested 2024-11-14T14:00:53,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853251 2024-11-14T14:00:53,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853251 2024-11-14T14:00:53,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=275, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853251) roll requested 2024-11-14T14:00:53,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853257 2024-11-14T14:00:53,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853251 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853257 2024-11-14T14:00:53,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=276, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853257) roll requested 2024-11-14T14:00:53,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853264 2024-11-14T14:00:53,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853264 2024-11-14T14:00:53,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=277, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853264) roll requested 2024-11-14T14:00:53,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853272 2024-11-14T14:00:53,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853272 2024-11-14T14:00:53,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,280 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=278, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853272) roll requested 2024-11-14T14:00:53,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853280 2024-11-14T14:00:53,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853280 2024-11-14T14:00:53,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=279, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,287 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,287 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853280) roll requested 2024-11-14T14:00:53,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853287 2024-11-14T14:00:53,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853287 2024-11-14T14:00:53,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=280, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853287) roll requested 2024-11-14T14:00:53,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853294 2024-11-14T14:00:53,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853287 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853294 2024-11-14T14:00:53,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=281, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853294) roll requested 2024-11-14T14:00:53,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853302 2024-11-14T14:00:53,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853302 2024-11-14T14:00:53,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=282, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853302) roll requested 2024-11-14T14:00:53,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853309 2024-11-14T14:00:53,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853309 2024-11-14T14:00:53,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=283, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853309) roll requested 2024-11-14T14:00:53,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853316 2024-11-14T14:00:53,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853316 2024-11-14T14:00:53,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,322 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=284, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,322 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853316) roll requested 2024-11-14T14:00:53,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853322 2024-11-14T14:00:53,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853322 2024-11-14T14:00:53,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=285, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853322) roll requested 2024-11-14T14:00:53,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853329 2024-11-14T14:00:53,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853322 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853329 2024-11-14T14:00:53,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=286, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853329) roll requested 2024-11-14T14:00:53,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853336 2024-11-14T14:00:53,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853336 2024-11-14T14:00:53,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,343 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=287, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,343 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853336) roll requested 2024-11-14T14:00:53,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853343 2024-11-14T14:00:53,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853343 2024-11-14T14:00:53,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=288, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853343) roll requested 2024-11-14T14:00:53,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853350 2024-11-14T14:00:53,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853343 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853350 2024-11-14T14:00:53,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=289, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853350) roll requested 2024-11-14T14:00:53,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853357 2024-11-14T14:00:53,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853357 2024-11-14T14:00:53,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=290, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853357) roll requested 2024-11-14T14:00:53,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853364 2024-11-14T14:00:53,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853364 2024-11-14T14:00:53,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=291, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853364) roll requested 2024-11-14T14:00:53,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853371 2024-11-14T14:00:53,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853371 2024-11-14T14:00:53,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=292, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853371) roll requested 2024-11-14T14:00:53,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853378 2024-11-14T14:00:53,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853378 2024-11-14T14:00:53,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=293, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853378) roll requested 2024-11-14T14:00:53,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853385 2024-11-14T14:00:53,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853385 2024-11-14T14:00:53,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=294, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853385) roll requested 2024-11-14T14:00:53,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853405 2024-11-14T14:00:53,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853405 2024-11-14T14:00:53,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=295, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853405) roll requested 2024-11-14T14:00:53,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853415 2024-11-14T14:00:53,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853415 2024-11-14T14:00:53,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=296, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853415) roll requested 2024-11-14T14:00:53,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853431 2024-11-14T14:00:53,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853431 2024-11-14T14:00:53,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=297, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,444 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,444 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853431) roll requested 2024-11-14T14:00:53,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853444 2024-11-14T14:00:53,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853444 2024-11-14T14:00:53,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=298, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,452 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853444) roll requested 2024-11-14T14:00:53,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853453 2024-11-14T14:00:53,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853444 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853453 2024-11-14T14:00:53,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,462 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=299, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853453) roll requested 2024-11-14T14:00:53,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853462 2024-11-14T14:00:53,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853453 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853462 2024-11-14T14:00:53,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=300, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853462) roll requested 2024-11-14T14:00:53,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853472 2024-11-14T14:00:53,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853472 2024-11-14T14:00:53,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=301, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853472) roll requested 2024-11-14T14:00:53,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853479 2024-11-14T14:00:53,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853479 2024-11-14T14:00:53,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,486 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,486 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=302, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853479) roll requested 2024-11-14T14:00:53,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853487 2024-11-14T14:00:53,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853487 2024-11-14T14:00:53,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,494 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=303, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,494 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,494 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853487) roll requested 2024-11-14T14:00:53,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853494 2024-11-14T14:00:53,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853487 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853494 2024-11-14T14:00:53,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,502 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=304, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,502 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853494) roll requested 2024-11-14T14:00:53,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853502 2024-11-14T14:00:53,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853494 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853502 2024-11-14T14:00:53,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,509 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=305, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853502) roll requested 2024-11-14T14:00:53,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853509 2024-11-14T14:00:53,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853502 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853509 2024-11-14T14:00:53,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,517 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,517 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=306, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853509) roll requested 2024-11-14T14:00:53,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853517 2024-11-14T14:00:53,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853517 2024-11-14T14:00:53,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=307, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,525 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853517) roll requested 2024-11-14T14:00:53,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853525 2024-11-14T14:00:53,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853525 2024-11-14T14:00:53,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=308, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,534 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853525) roll requested 2024-11-14T14:00:53,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853534 2024-11-14T14:00:53,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853534 2024-11-14T14:00:53,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,540 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=309, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853534) roll requested 2024-11-14T14:00:53,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853541 2024-11-14T14:00:53,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853534 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853541 2024-11-14T14:00:53,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=310, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853541) roll requested 2024-11-14T14:00:53,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853548 2024-11-14T14:00:53,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853548 2024-11-14T14:00:53,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,554 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=311, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853548) roll requested 2024-11-14T14:00:53,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853555 2024-11-14T14:00:53,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853555 2024-11-14T14:00:53,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=312, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853555) roll requested 2024-11-14T14:00:53,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853561 2024-11-14T14:00:53,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853561 2024-11-14T14:00:53,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,567 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,567 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=313, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,567 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853561) roll requested 2024-11-14T14:00:53,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853567 2024-11-14T14:00:53,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853567 2024-11-14T14:00:53,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=314, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853567) roll requested 2024-11-14T14:00:53,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853573 2024-11-14T14:00:53,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853567 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853573 2024-11-14T14:00:53,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,579 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=315, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853573) roll requested 2024-11-14T14:00:53,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853580 2024-11-14T14:00:53,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853580 2024-11-14T14:00:53,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,587 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=316, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853580) roll requested 2024-11-14T14:00:53,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853587 2024-11-14T14:00:53,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853587 2024-11-14T14:00:53,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=317, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853587) roll requested 2024-11-14T14:00:53,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853593 2024-11-14T14:00:53,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853593 2024-11-14T14:00:53,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=318, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853593) roll requested 2024-11-14T14:00:53,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853600 2024-11-14T14:00:53,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853600 2024-11-14T14:00:53,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,606 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,606 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=319, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,606 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853600) roll requested 2024-11-14T14:00:53,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853606 2024-11-14T14:00:53,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853606 2024-11-14T14:00:53,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=320, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853606) roll requested 2024-11-14T14:00:53,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853621 2024-11-14T14:00:53,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853606 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853621 2024-11-14T14:00:53,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,627 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,627 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=321, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853621) roll requested 2024-11-14T14:00:53,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853628 2024-11-14T14:00:53,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853628 2024-11-14T14:00:53,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,634 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,634 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=322, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,634 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853628) roll requested 2024-11-14T14:00:53,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853634 2024-11-14T14:00:53,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853634 2024-11-14T14:00:53,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=323, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853634) roll requested 2024-11-14T14:00:53,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853640 2024-11-14T14:00:53,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853640 2024-11-14T14:00:53,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,647 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,647 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=324, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853640) roll requested 2024-11-14T14:00:53,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853647 2024-11-14T14:00:53,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853647 2024-11-14T14:00:53,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=325, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853647) roll requested 2024-11-14T14:00:53,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853654 2024-11-14T14:00:53,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853654 2024-11-14T14:00:53,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=326, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853654) roll requested 2024-11-14T14:00:53,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853661 2024-11-14T14:00:53,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853661 2024-11-14T14:00:53,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,668 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=327, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,668 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853661) roll requested 2024-11-14T14:00:53,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853668 2024-11-14T14:00:53,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853661 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853668 2024-11-14T14:00:53,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,675 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=328, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,675 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853668) roll requested 2024-11-14T14:00:53,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853675 2024-11-14T14:00:53,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853668 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853675 2024-11-14T14:00:53,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,682 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=329, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,682 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853675) roll requested 2024-11-14T14:00:53,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853682 2024-11-14T14:00:53,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853682 2024-11-14T14:00:53,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=330, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853682) roll requested 2024-11-14T14:00:53,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853690 2024-11-14T14:00:53,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853682 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853690 2024-11-14T14:00:53,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=331, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853690) roll requested 2024-11-14T14:00:53,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853696 2024-11-14T14:00:53,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853690 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853696 2024-11-14T14:00:53,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,703 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=332, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853696) roll requested 2024-11-14T14:00:53,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853703 2024-11-14T14:00:53,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853703 2024-11-14T14:00:53,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,710 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,710 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=333, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,710 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853703) roll requested 2024-11-14T14:00:53,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853710 2024-11-14T14:00:53,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853710 2024-11-14T14:00:53,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=334, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853710) roll requested 2024-11-14T14:00:53,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853717 2024-11-14T14:00:53,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853710 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853717 2024-11-14T14:00:53,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=335, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853717) roll requested 2024-11-14T14:00:53,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853725 2024-11-14T14:00:53,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853725 2024-11-14T14:00:53,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=336, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853725) roll requested 2024-11-14T14:00:53,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853732 2024-11-14T14:00:53,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853732 2024-11-14T14:00:53,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,739 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=337, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,739 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853732) roll requested 2024-11-14T14:00:53,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853739 2024-11-14T14:00:53,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853739 2024-11-14T14:00:53,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=338, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853739) roll requested 2024-11-14T14:00:53,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853745 2024-11-14T14:00:53,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853739 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853745 2024-11-14T14:00:53,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=339, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853745) roll requested 2024-11-14T14:00:53,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853752 2024-11-14T14:00:53,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853752 2024-11-14T14:00:53,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=340, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853752) roll requested 2024-11-14T14:00:53,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853759 2024-11-14T14:00:53,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853759 2024-11-14T14:00:53,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=341, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853759) roll requested 2024-11-14T14:00:53,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853767 2024-11-14T14:00:53,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853767 2024-11-14T14:00:53,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=342, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853767) roll requested 2024-11-14T14:00:53,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853777 2024-11-14T14:00:53,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853777 2024-11-14T14:00:53,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=343, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853777) roll requested 2024-11-14T14:00:53,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853784 2024-11-14T14:00:53,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853784 2024-11-14T14:00:53,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=344, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853784) roll requested 2024-11-14T14:00:53,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853792 2024-11-14T14:00:53,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853792 2024-11-14T14:00:53,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=345, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853792) roll requested 2024-11-14T14:00:53,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853799 2024-11-14T14:00:53,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853799 2024-11-14T14:00:53,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=346, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853799) roll requested 2024-11-14T14:00:53,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853806 2024-11-14T14:00:53,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853806 2024-11-14T14:00:53,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=347, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853806) roll requested 2024-11-14T14:00:53,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853813 2024-11-14T14:00:53,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853813 2024-11-14T14:00:53,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=348, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853813) roll requested 2024-11-14T14:00:53,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853820 2024-11-14T14:00:53,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853820 2024-11-14T14:00:53,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,826 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,826 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=349, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,826 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853820) roll requested 2024-11-14T14:00:53,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853826 2024-11-14T14:00:53,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853826 2024-11-14T14:00:53,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,833 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,834 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=350, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,834 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853826) roll requested 2024-11-14T14:00:53,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853834 2024-11-14T14:00:53,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853826 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853834 2024-11-14T14:00:53,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=351, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853834) roll requested 2024-11-14T14:00:53,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853840 2024-11-14T14:00:53,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853834 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853840 2024-11-14T14:00:53,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=352, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853840) roll requested 2024-11-14T14:00:53,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853847 2024-11-14T14:00:53,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853847 2024-11-14T14:00:53,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=353, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853847) roll requested 2024-11-14T14:00:53,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853854 2024-11-14T14:00:53,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853854 2024-11-14T14:00:53,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=354, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,861 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853854) roll requested 2024-11-14T14:00:53,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853861 2024-11-14T14:00:53,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853861 2024-11-14T14:00:53,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=355, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,868 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853861) roll requested 2024-11-14T14:00:53,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853868 2024-11-14T14:00:53,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853868 2024-11-14T14:00:53,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,875 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=356, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,876 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853868) roll requested 2024-11-14T14:00:53,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853876 2024-11-14T14:00:53,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853868 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853876 2024-11-14T14:00:53,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,882 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=357, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,883 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853876) roll requested 2024-11-14T14:00:53,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853883 2024-11-14T14:00:53,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853883 2024-11-14T14:00:53,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,889 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=358, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,889 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853883) roll requested 2024-11-14T14:00:53,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853889 2024-11-14T14:00:53,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853889 2024-11-14T14:00:53,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,896 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=359, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,896 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853889) roll requested 2024-11-14T14:00:53,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853896 2024-11-14T14:00:53,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853896 2024-11-14T14:00:53,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,903 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=360, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,903 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853896) roll requested 2024-11-14T14:00:53,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853903 2024-11-14T14:00:53,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853903 2024-11-14T14:00:53,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=361, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,910 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853903) roll requested 2024-11-14T14:00:53,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853910 2024-11-14T14:00:53,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853903 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853910 2024-11-14T14:00:53,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=362, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853910) roll requested 2024-11-14T14:00:53,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853917 2024-11-14T14:00:53,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853917 2024-11-14T14:00:53,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=363, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853917) roll requested 2024-11-14T14:00:53,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853924 2024-11-14T14:00:53,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853924 2024-11-14T14:00:53,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=364, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853924) roll requested 2024-11-14T14:00:53,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853931 2024-11-14T14:00:53,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853931 2024-11-14T14:00:53,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=365, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853931) roll requested 2024-11-14T14:00:53,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853938 2024-11-14T14:00:53,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853938 2024-11-14T14:00:53,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=366, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853938) roll requested 2024-11-14T14:00:53,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853945 2024-11-14T14:00:53,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853945 2024-11-14T14:00:53,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=367, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853945) roll requested 2024-11-14T14:00:53,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853952 2024-11-14T14:00:53,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853952 2024-11-14T14:00:53,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=368, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853952) roll requested 2024-11-14T14:00:53,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853959 2024-11-14T14:00:53,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853959 2024-11-14T14:00:53,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=369, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853959) roll requested 2024-11-14T14:00:53,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853966 2024-11-14T14:00:53,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853966 2024-11-14T14:00:53,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,974 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=370, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,974 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853966) roll requested 2024-11-14T14:00:53,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853974 2024-11-14T14:00:53,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853974 2024-11-14T14:00:53,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,981 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,981 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=371, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853974) roll requested 2024-11-14T14:00:53,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853981 2024-11-14T14:00:53,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853974 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853981 2024-11-14T14:00:53,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,988 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=372, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,988 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853981) roll requested 2024-11-14T14:00:53,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853988 2024-11-14T14:00:53,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:53,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853988 2024-11-14T14:00:53,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:53,995 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=373, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:53,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:53,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:53,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853988) roll requested 2024-11-14T14:00:53,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592853995 2024-11-14T14:00:54,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853988 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853995 2024-11-14T14:00:54,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,002 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=374, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592853995) roll requested 2024-11-14T14:00:54,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854002 2024-11-14T14:00:54,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592853995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854002 2024-11-14T14:00:54,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=375, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854002) roll requested 2024-11-14T14:00:54,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854009 2024-11-14T14:00:54,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854009 2024-11-14T14:00:54,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,016 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,016 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=376, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,016 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854009) roll requested 2024-11-14T14:00:54,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854016 2024-11-14T14:00:54,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854016 2024-11-14T14:00:54,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,023 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,023 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=377, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,023 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854016) roll requested 2024-11-14T14:00:54,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854023 2024-11-14T14:00:54,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854016 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854023 2024-11-14T14:00:54,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=378, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854023) roll requested 2024-11-14T14:00:54,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854029 2024-11-14T14:00:54,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854029 2024-11-14T14:00:54,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=379, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854029) roll requested 2024-11-14T14:00:54,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854036 2024-11-14T14:00:54,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854036 2024-11-14T14:00:54,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,043 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=380, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854036) roll requested 2024-11-14T14:00:54,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854043 2024-11-14T14:00:54,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854043 2024-11-14T14:00:54,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,050 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=381, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,050 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,050 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854043) roll requested 2024-11-14T14:00:54,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854050 2024-11-14T14:00:54,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854050 2024-11-14T14:00:54,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=382, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854050) roll requested 2024-11-14T14:00:54,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854057 2024-11-14T14:00:54,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854057 2024-11-14T14:00:54,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=383, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,064 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854057) roll requested 2024-11-14T14:00:54,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854064 2024-11-14T14:00:54,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854064 2024-11-14T14:00:54,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=384, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854064) roll requested 2024-11-14T14:00:54,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854072 2024-11-14T14:00:54,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854064 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854072 2024-11-14T14:00:54,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=385, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854072) roll requested 2024-11-14T14:00:54,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854079 2024-11-14T14:00:54,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854079 2024-11-14T14:00:54,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=386, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854079) roll requested 2024-11-14T14:00:54,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854085 2024-11-14T14:00:54,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854085 2024-11-14T14:00:54,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=387, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854085) roll requested 2024-11-14T14:00:54,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854092 2024-11-14T14:00:54,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854092 2024-11-14T14:00:54,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=388, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854092) roll requested 2024-11-14T14:00:54,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854098 2024-11-14T14:00:54,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854098 2024-11-14T14:00:54,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=389, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,107 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854098) roll requested 2024-11-14T14:00:54,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854108 2024-11-14T14:00:54,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854108 2024-11-14T14:00:54,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,114 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=390, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,114 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854108) roll requested 2024-11-14T14:00:54,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854114 2024-11-14T14:00:54,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854114 2024-11-14T14:00:54,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=391, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854114) roll requested 2024-11-14T14:00:54,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854122 2024-11-14T14:00:54,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854114 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854122 2024-11-14T14:00:54,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=392, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854122) roll requested 2024-11-14T14:00:54,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854128 2024-11-14T14:00:54,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854122 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854128 2024-11-14T14:00:54,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=393, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854128) roll requested 2024-11-14T14:00:54,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854135 2024-11-14T14:00:54,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854128 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854135 2024-11-14T14:00:54,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,142 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=394, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,142 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854135) roll requested 2024-11-14T14:00:54,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854142 2024-11-14T14:00:54,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854142 2024-11-14T14:00:54,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=395, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854142) roll requested 2024-11-14T14:00:54,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854149 2024-11-14T14:00:54,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854142 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854149 2024-11-14T14:00:54,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=396, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854149) roll requested 2024-11-14T14:00:54,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854156 2024-11-14T14:00:54,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854156 2024-11-14T14:00:54,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,163 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,163 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=397, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,163 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854156) roll requested 2024-11-14T14:00:54,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854163 2024-11-14T14:00:54,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854163 2024-11-14T14:00:54,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,169 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,169 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=398, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854163) roll requested 2024-11-14T14:00:54,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854170 2024-11-14T14:00:54,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854163 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854170 2024-11-14T14:00:54,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=399, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854170) roll requested 2024-11-14T14:00:54,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854176 2024-11-14T14:00:54,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854170 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854176 2024-11-14T14:00:54,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=400, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854176) roll requested 2024-11-14T14:00:54,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854183 2024-11-14T14:00:54,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854176 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854183 2024-11-14T14:00:54,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,191 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,191 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=401, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,191 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854183) roll requested 2024-11-14T14:00:54,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854191 2024-11-14T14:00:54,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854183 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854191 2024-11-14T14:00:54,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=402, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854191) roll requested 2024-11-14T14:00:54,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854197 2024-11-14T14:00:54,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854191 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854197 2024-11-14T14:00:54,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=403, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854197) roll requested 2024-11-14T14:00:54,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854205 2024-11-14T14:00:54,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854205 2024-11-14T14:00:54,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,212 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,212 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=404, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854205) roll requested 2024-11-14T14:00:54,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854212 2024-11-14T14:00:54,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854212 2024-11-14T14:00:54,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=405, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854212) roll requested 2024-11-14T14:00:54,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854219 2024-11-14T14:00:54,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854219 2024-11-14T14:00:54,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=406, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854219) roll requested 2024-11-14T14:00:54,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854225 2024-11-14T14:00:54,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854225 2024-11-14T14:00:54,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=407, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854225) roll requested 2024-11-14T14:00:54,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854233 2024-11-14T14:00:54,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854233 2024-11-14T14:00:54,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=408, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854233) roll requested 2024-11-14T14:00:54,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854240 2024-11-14T14:00:54,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854240 2024-11-14T14:00:54,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=409, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854240) roll requested 2024-11-14T14:00:54,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854247 2024-11-14T14:00:54,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854247 2024-11-14T14:00:54,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=410, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854247) roll requested 2024-11-14T14:00:54,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854255 2024-11-14T14:00:54,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854255 2024-11-14T14:00:54,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=411, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854255) roll requested 2024-11-14T14:00:54,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854264 2024-11-14T14:00:54,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854264 2024-11-14T14:00:54,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=412, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854264) roll requested 2024-11-14T14:00:54,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854271 2024-11-14T14:00:54,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854271 2024-11-14T14:00:54,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,278 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=413, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,278 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854271) roll requested 2024-11-14T14:00:54,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854278 2024-11-14T14:00:54,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854278 2024-11-14T14:00:54,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=414, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854278) roll requested 2024-11-14T14:00:54,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854285 2024-11-14T14:00:54,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854285 2024-11-14T14:00:54,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=415, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854285) roll requested 2024-11-14T14:00:54,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854292 2024-11-14T14:00:54,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854292 2024-11-14T14:00:54,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,299 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=416, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854292) roll requested 2024-11-14T14:00:54,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854299 2024-11-14T14:00:54,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854299 2024-11-14T14:00:54,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,306 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=417, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,306 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854299) roll requested 2024-11-14T14:00:54,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854306 2024-11-14T14:00:54,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854306 2024-11-14T14:00:54,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=418, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854306) roll requested 2024-11-14T14:00:54,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854313 2024-11-14T14:00:54,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854306 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854313 2024-11-14T14:00:54,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=419, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854313) roll requested 2024-11-14T14:00:54,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854320 2024-11-14T14:00:54,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854320 2024-11-14T14:00:54,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,327 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=420, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,327 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854320) roll requested 2024-11-14T14:00:54,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854327 2024-11-14T14:00:54,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854327 2024-11-14T14:00:54,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=421, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854327) roll requested 2024-11-14T14:00:54,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854334 2024-11-14T14:00:54,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854334 2024-11-14T14:00:54,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=422, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854334) roll requested 2024-11-14T14:00:54,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854340 2024-11-14T14:00:54,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854340 2024-11-14T14:00:54,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,347 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,347 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=423, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,347 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854340) roll requested 2024-11-14T14:00:54,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854347 2024-11-14T14:00:54,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854347 2024-11-14T14:00:54,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=424, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854347) roll requested 2024-11-14T14:00:54,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854355 2024-11-14T14:00:54,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854347 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854355 2024-11-14T14:00:54,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=425, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854355) roll requested 2024-11-14T14:00:54,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854362 2024-11-14T14:00:54,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854362 2024-11-14T14:00:54,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=426, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854362) roll requested 2024-11-14T14:00:54,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854369 2024-11-14T14:00:54,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854362 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854369 2024-11-14T14:00:54,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,376 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,376 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=427, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,376 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854369) roll requested 2024-11-14T14:00:54,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854376 2024-11-14T14:00:54,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854376 2024-11-14T14:00:54,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,383 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,383 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=428, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854376) roll requested 2024-11-14T14:00:54,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854384 2024-11-14T14:00:54,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854376 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854384 2024-11-14T14:00:54,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=429, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854384) roll requested 2024-11-14T14:00:54,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854390 2024-11-14T14:00:54,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854384 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854390 2024-11-14T14:00:54,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=430, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854390) roll requested 2024-11-14T14:00:54,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854397 2024-11-14T14:00:54,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854397 2024-11-14T14:00:54,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=431, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854397) roll requested 2024-11-14T14:00:54,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854404 2024-11-14T14:00:54,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854404 2024-11-14T14:00:54,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,412 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,412 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=432, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854404) roll requested 2024-11-14T14:00:54,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854412 2024-11-14T14:00:54,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854412 2024-11-14T14:00:54,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,419 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,419 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=433, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,419 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854412) roll requested 2024-11-14T14:00:54,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854419 2024-11-14T14:00:54,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854419 2024-11-14T14:00:54,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,426 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,426 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=434, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,426 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854419) roll requested 2024-11-14T14:00:54,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854427 2024-11-14T14:00:54,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854419 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854427 2024-11-14T14:00:54,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,433 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,433 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=435, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854427) roll requested 2024-11-14T14:00:54,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854434 2024-11-14T14:00:54,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854434 2024-11-14T14:00:54,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=436, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854434) roll requested 2024-11-14T14:00:54,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854442 2024-11-14T14:00:54,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854442 2024-11-14T14:00:54,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=437, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,449 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854442) roll requested 2024-11-14T14:00:54,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854449 2024-11-14T14:00:54,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854449 2024-11-14T14:00:54,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=438, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854449) roll requested 2024-11-14T14:00:54,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854459 2024-11-14T14:00:54,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854449 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854459 2024-11-14T14:00:54,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=439, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854459) roll requested 2024-11-14T14:00:54,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854466 2024-11-14T14:00:54,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854466 2024-11-14T14:00:54,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=440, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854466) roll requested 2024-11-14T14:00:54,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854474 2024-11-14T14:00:54,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854474 2024-11-14T14:00:54,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,481 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=441, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,481 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854474) roll requested 2024-11-14T14:00:54,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854481 2024-11-14T14:00:54,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854481 2024-11-14T14:00:54,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=442, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854481) roll requested 2024-11-14T14:00:54,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854488 2024-11-14T14:00:54,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854481 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854488 2024-11-14T14:00:54,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,495 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,495 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=443, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,495 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854488) roll requested 2024-11-14T14:00:54,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854495 2024-11-14T14:00:54,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854495 2024-11-14T14:00:54,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=444, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854495) roll requested 2024-11-14T14:00:54,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854501 2024-11-14T14:00:54,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854495 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854501 2024-11-14T14:00:54,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=445, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854501) roll requested 2024-11-14T14:00:54,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854508 2024-11-14T14:00:54,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854508 2024-11-14T14:00:54,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,515 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=446, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,515 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854508) roll requested 2024-11-14T14:00:54,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854515 2024-11-14T14:00:54,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854515 2024-11-14T14:00:54,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,521 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=447, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854515) roll requested 2024-11-14T14:00:54,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854522 2024-11-14T14:00:54,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854522 2024-11-14T14:00:54,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=448, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854522) roll requested 2024-11-14T14:00:54,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854528 2024-11-14T14:00:54,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854528 2024-11-14T14:00:54,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=449, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854528) roll requested 2024-11-14T14:00:54,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854536 2024-11-14T14:00:54,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854536 2024-11-14T14:00:54,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=450, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854536) roll requested 2024-11-14T14:00:54,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854542 2024-11-14T14:00:54,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854542 2024-11-14T14:00:54,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,549 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=451, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,549 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854542) roll requested 2024-11-14T14:00:54,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854549 2024-11-14T14:00:54,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854549 2024-11-14T14:00:54,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,560 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,560 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=452, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,560 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854549) roll requested 2024-11-14T14:00:54,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854560 2024-11-14T14:00:54,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854560 2024-11-14T14:00:54,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,567 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,567 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=453, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,567 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854560) roll requested 2024-11-14T14:00:54,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854567 2024-11-14T14:00:54,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854560 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854567 2024-11-14T14:00:54,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=454, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854567) roll requested 2024-11-14T14:00:54,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854577 2024-11-14T14:00:54,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854567 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854577 2024-11-14T14:00:54,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,585 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,585 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=455, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854577) roll requested 2024-11-14T14:00:54,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854585 2024-11-14T14:00:54,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854585 2024-11-14T14:00:54,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=456, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854585) roll requested 2024-11-14T14:00:54,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854592 2024-11-14T14:00:54,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854592 2024-11-14T14:00:54,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=457, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854592) roll requested 2024-11-14T14:00:54,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854600 2024-11-14T14:00:54,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854600 2024-11-14T14:00:54,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=458, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854600) roll requested 2024-11-14T14:00:54,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854608 2024-11-14T14:00:54,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854608 2024-11-14T14:00:54,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,617 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=459, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854608) roll requested 2024-11-14T14:00:54,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854617 2024-11-14T14:00:54,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854617 2024-11-14T14:00:54,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,625 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=460, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854617) roll requested 2024-11-14T14:00:54,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854625 2024-11-14T14:00:54,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854625 2024-11-14T14:00:54,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=461, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854625) roll requested 2024-11-14T14:00:54,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854633 2024-11-14T14:00:54,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854633 2024-11-14T14:00:54,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=462, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854633) roll requested 2024-11-14T14:00:54,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854643 2024-11-14T14:00:54,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854643 2024-11-14T14:00:54,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=463, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854643) roll requested 2024-11-14T14:00:54,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854654 2024-11-14T14:00:54,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854643 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854654 2024-11-14T14:00:54,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=464, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854654) roll requested 2024-11-14T14:00:54,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854661 2024-11-14T14:00:54,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854661 2024-11-14T14:00:54,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=465, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,669 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854661) roll requested 2024-11-14T14:00:54,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854669 2024-11-14T14:00:54,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854661 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854669 2024-11-14T14:00:54,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=466, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854669) roll requested 2024-11-14T14:00:54,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854676 2024-11-14T14:00:54,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854669 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854676 2024-11-14T14:00:54,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=467, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854676) roll requested 2024-11-14T14:00:54,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854689 2024-11-14T14:00:54,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854689 2024-11-14T14:00:54,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,704 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,704 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=468, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,704 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854689) roll requested 2024-11-14T14:00:54,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854705 2024-11-14T14:00:54,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854705 2024-11-14T14:00:54,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,711 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,711 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=469, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,711 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854705) roll requested 2024-11-14T14:00:54,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854711 2024-11-14T14:00:54,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854705 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854711 2024-11-14T14:00:54,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,718 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,718 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=470, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,719 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854711) roll requested 2024-11-14T14:00:54,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854719 2024-11-14T14:00:54,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854711 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854719 2024-11-14T14:00:54,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=471, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,728 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854719) roll requested 2024-11-14T14:00:54,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854728 2024-11-14T14:00:54,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854719 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854728 2024-11-14T14:00:54,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,735 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=472, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854728) roll requested 2024-11-14T14:00:54,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854735 2024-11-14T14:00:54,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854735 2024-11-14T14:00:54,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,742 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=473, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,743 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854735) roll requested 2024-11-14T14:00:54,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854743 2024-11-14T14:00:54,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854743 2024-11-14T14:00:54,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=474, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854743) roll requested 2024-11-14T14:00:54,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854753 2024-11-14T14:00:54,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854753 2024-11-14T14:00:54,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=475, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854753) roll requested 2024-11-14T14:00:54,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854760 2024-11-14T14:00:54,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854760 2024-11-14T14:00:54,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=476, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854760) roll requested 2024-11-14T14:00:54,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854769 2024-11-14T14:00:54,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854769 2024-11-14T14:00:54,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=477, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854769) roll requested 2024-11-14T14:00:54,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854776 2024-11-14T14:00:54,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854776 2024-11-14T14:00:54,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,783 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=478, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,783 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854776) roll requested 2024-11-14T14:00:54,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854783 2024-11-14T14:00:54,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854783 2024-11-14T14:00:54,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=479, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854783) roll requested 2024-11-14T14:00:54,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854791 2024-11-14T14:00:54,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854783 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854791 2024-11-14T14:00:54,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=480, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,801 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854791) roll requested 2024-11-14T14:00:54,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854801 2024-11-14T14:00:54,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854801 2024-11-14T14:00:54,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=481, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854801) roll requested 2024-11-14T14:00:54,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854808 2024-11-14T14:00:54,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854808 2024-11-14T14:00:54,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=482, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854808) roll requested 2024-11-14T14:00:54,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854816 2024-11-14T14:00:54,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854816 2024-11-14T14:00:54,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,823 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=483, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854816) roll requested 2024-11-14T14:00:54,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854823 2024-11-14T14:00:54,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854823 2024-11-14T14:00:54,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=484, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854823) roll requested 2024-11-14T14:00:54,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854831 2024-11-14T14:00:54,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854831 2024-11-14T14:00:54,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,841 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=485, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854831) roll requested 2024-11-14T14:00:54,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854841 2024-11-14T14:00:54,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854841 2024-11-14T14:00:54,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,848 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,848 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=486, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,848 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854841) roll requested 2024-11-14T14:00:54,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854848 2024-11-14T14:00:54,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854848 2024-11-14T14:00:54,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=487, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854848) roll requested 2024-11-14T14:00:54,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854856 2024-11-14T14:00:54,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854856 2024-11-14T14:00:54,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,863 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=488, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854856) roll requested 2024-11-14T14:00:54,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854864 2024-11-14T14:00:54,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854864 2024-11-14T14:00:54,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=489, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854864) roll requested 2024-11-14T14:00:54,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854871 2024-11-14T14:00:54,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854871 2024-11-14T14:00:54,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=490, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854871) roll requested 2024-11-14T14:00:54,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854881 2024-11-14T14:00:54,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854881 2024-11-14T14:00:54,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,888 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=491, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854881) roll requested 2024-11-14T14:00:54,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854888 2024-11-14T14:00:54,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854888 2024-11-14T14:00:54,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,895 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,896 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=492, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,896 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854888) roll requested 2024-11-14T14:00:54,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854896 2024-11-14T14:00:54,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854896 2024-11-14T14:00:54,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,903 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=493, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,903 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854896) roll requested 2024-11-14T14:00:54,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854903 2024-11-14T14:00:54,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854903 2024-11-14T14:00:54,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,910 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=494, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,910 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854903) roll requested 2024-11-14T14:00:54,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854910 2024-11-14T14:00:54,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854903 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854910 2024-11-14T14:00:54,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=495, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,918 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854910) roll requested 2024-11-14T14:00:54,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854918 2024-11-14T14:00:54,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854918 2024-11-14T14:00:54,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=496, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854918) roll requested 2024-11-14T14:00:54,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854924 2024-11-14T14:00:54,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854924 2024-11-14T14:00:54,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=497, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854924) roll requested 2024-11-14T14:00:54,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854932 2024-11-14T14:00:54,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854932 2024-11-14T14:00:54,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=498, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854932) roll requested 2024-11-14T14:00:54,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854941 2024-11-14T14:00:54,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854932 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854941 2024-11-14T14:00:54,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=499, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,948 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854941) roll requested 2024-11-14T14:00:54,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854948 2024-11-14T14:00:54,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854948 2024-11-14T14:00:54,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,957 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,957 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=500, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,957 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854948) roll requested 2024-11-14T14:00:54,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854957 2024-11-14T14:00:54,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854948 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854957 2024-11-14T14:00:54,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=501, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854957) roll requested 2024-11-14T14:00:54,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854967 2024-11-14T14:00:54,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854957 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854967 2024-11-14T14:00:54,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,974 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=502, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,975 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854967) roll requested 2024-11-14T14:00:54,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854975 2024-11-14T14:00:54,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854975 2024-11-14T14:00:54,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,981 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,981 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=503, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854975) roll requested 2024-11-14T14:00:54,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854981 2024-11-14T14:00:54,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854981 2024-11-14T14:00:54,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=504, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854981) roll requested 2024-11-14T14:00:54,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854989 2024-11-14T14:00:54,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:54,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854989 2024-11-14T14:00:54,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:54,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:54,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=505, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:54,996 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:54,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854989) roll requested 2024-11-14T14:00:54,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592854996 2024-11-14T14:00:55,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854996 2024-11-14T14:00:55,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=506, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592854996) roll requested 2024-11-14T14:00:55,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855003 2024-11-14T14:00:55,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592854996 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855003 2024-11-14T14:00:55,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,010 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=507, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855003) roll requested 2024-11-14T14:00:55,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855010 2024-11-14T14:00:55,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855010 2024-11-14T14:00:55,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=508, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855010) roll requested 2024-11-14T14:00:55,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855018 2024-11-14T14:00:55,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855018 2024-11-14T14:00:55,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=509, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855018) roll requested 2024-11-14T14:00:55,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855025 2024-11-14T14:00:55,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855025 2024-11-14T14:00:55,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,031 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=510, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855025) roll requested 2024-11-14T14:00:55,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855032 2024-11-14T14:00:55,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855032 2024-11-14T14:00:55,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,039 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=511, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,039 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,039 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855032) roll requested 2024-11-14T14:00:55,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855039 2024-11-14T14:00:55,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855039 2024-11-14T14:00:55,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=512, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,047 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855039) roll requested 2024-11-14T14:00:55,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855047 2024-11-14T14:00:55,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855039 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855047 2024-11-14T14:00:55,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=513, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855047) roll requested 2024-11-14T14:00:55,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855054 2024-11-14T14:00:55,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855047 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855054 2024-11-14T14:00:55,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=514, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855054) roll requested 2024-11-14T14:00:55,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855061 2024-11-14T14:00:55,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855061 2024-11-14T14:00:55,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=515, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855061) roll requested 2024-11-14T14:00:55,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855067 2024-11-14T14:00:55,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855067 2024-11-14T14:00:55,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,074 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,074 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=516, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,074 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855067) roll requested 2024-11-14T14:00:55,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855075 2024-11-14T14:00:55,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855075 2024-11-14T14:00:55,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=517, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855075) roll requested 2024-11-14T14:00:55,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855085 2024-11-14T14:00:55,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855075 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855085 2024-11-14T14:00:55,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,095 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,095 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=518, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855085) roll requested 2024-11-14T14:00:55,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855096 2024-11-14T14:00:55,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855096 2024-11-14T14:00:55,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=519, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855096) roll requested 2024-11-14T14:00:55,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855105 2024-11-14T14:00:55,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855105 2024-11-14T14:00:55,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=520, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855105) roll requested 2024-11-14T14:00:55,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855113 2024-11-14T14:00:55,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855113 2024-11-14T14:00:55,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=521, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855113) roll requested 2024-11-14T14:00:55,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855120 2024-11-14T14:00:55,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855120 2024-11-14T14:00:55,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,127 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,127 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=522, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,127 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855120) roll requested 2024-11-14T14:00:55,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855127 2024-11-14T14:00:55,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855127 2024-11-14T14:00:55,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,134 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,134 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=523, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,134 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855127) roll requested 2024-11-14T14:00:55,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855135 2024-11-14T14:00:55,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855127 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855135 2024-11-14T14:00:55,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=524, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,142 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855135) roll requested 2024-11-14T14:00:55,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855142 2024-11-14T14:00:55,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855142 2024-11-14T14:00:55,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=525, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855142) roll requested 2024-11-14T14:00:55,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855149 2024-11-14T14:00:55,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855142 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855149 2024-11-14T14:00:55,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=526, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855149) roll requested 2024-11-14T14:00:55,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855157 2024-11-14T14:00:55,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855157 2024-11-14T14:00:55,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,164 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,164 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=527, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,164 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855157) roll requested 2024-11-14T14:00:55,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855164 2024-11-14T14:00:55,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855164 2024-11-14T14:00:55,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,171 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,171 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=528, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,171 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855164) roll requested 2024-11-14T14:00:55,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855171 2024-11-14T14:00:55,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855164 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855171 2024-11-14T14:00:55,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=529, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855171) roll requested 2024-11-14T14:00:55,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855178 2024-11-14T14:00:55,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855178 2024-11-14T14:00:55,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=530, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855178) roll requested 2024-11-14T14:00:55,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855185 2024-11-14T14:00:55,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855185 2024-11-14T14:00:55,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,192 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=531, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855185) roll requested 2024-11-14T14:00:55,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855192 2024-11-14T14:00:55,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855192 2024-11-14T14:00:55,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,199 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,199 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=532, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,199 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855192) roll requested 2024-11-14T14:00:55,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855199 2024-11-14T14:00:55,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855192 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855199 2024-11-14T14:00:55,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,206 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,206 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=533, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,206 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855199) roll requested 2024-11-14T14:00:55,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855206 2024-11-14T14:00:55,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855199 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855206 2024-11-14T14:00:55,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,213 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=534, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,213 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855206) roll requested 2024-11-14T14:00:55,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855213 2024-11-14T14:00:55,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855206 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855213 2024-11-14T14:00:55,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,220 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,220 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=535, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,220 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855213) roll requested 2024-11-14T14:00:55,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855220 2024-11-14T14:00:55,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855213 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855220 2024-11-14T14:00:55,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,227 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,227 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=536, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,227 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855220) roll requested 2024-11-14T14:00:55,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855227 2024-11-14T14:00:55,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855220 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855227 2024-11-14T14:00:55,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,241 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=537, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855227) roll requested 2024-11-14T14:00:55,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855241 2024-11-14T14:00:55,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855227 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855241 2024-11-14T14:00:55,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,248 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=538, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855241) roll requested 2024-11-14T14:00:55,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855248 2024-11-14T14:00:55,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855248 2024-11-14T14:00:55,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=539, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855248) roll requested 2024-11-14T14:00:55,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855256 2024-11-14T14:00:55,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855256 2024-11-14T14:00:55,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=540, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855256) roll requested 2024-11-14T14:00:55,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855265 2024-11-14T14:00:55,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855256 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855265 2024-11-14T14:00:55,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=541, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855265) roll requested 2024-11-14T14:00:55,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855272 2024-11-14T14:00:55,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855272 2024-11-14T14:00:55,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=542, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855272) roll requested 2024-11-14T14:00:55,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855280 2024-11-14T14:00:55,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855280 2024-11-14T14:00:55,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,287 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=543, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,287 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855280) roll requested 2024-11-14T14:00:55,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855287 2024-11-14T14:00:55,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855287 2024-11-14T14:00:55,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=544, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855287) roll requested 2024-11-14T14:00:55,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855294 2024-11-14T14:00:55,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855287 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855294 2024-11-14T14:00:55,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=545, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855294) roll requested 2024-11-14T14:00:55,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855301 2024-11-14T14:00:55,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855301 2024-11-14T14:00:55,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=546, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855301) roll requested 2024-11-14T14:00:55,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855308 2024-11-14T14:00:55,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855308 2024-11-14T14:00:55,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,315 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,315 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=547, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,315 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855308) roll requested 2024-11-14T14:00:55,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855315 2024-11-14T14:00:55,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855315 2024-11-14T14:00:55,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,322 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=548, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,322 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855315) roll requested 2024-11-14T14:00:55,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855322 2024-11-14T14:00:55,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855315 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855322 2024-11-14T14:00:55,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=549, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855322) roll requested 2024-11-14T14:00:55,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855329 2024-11-14T14:00:55,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855322 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855329 2024-11-14T14:00:55,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=550, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855329) roll requested 2024-11-14T14:00:55,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855336 2024-11-14T14:00:55,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855336 2024-11-14T14:00:55,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,343 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=551, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,343 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855336) roll requested 2024-11-14T14:00:55,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855343 2024-11-14T14:00:55,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855343 2024-11-14T14:00:55,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=552, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855343) roll requested 2024-11-14T14:00:55,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855350 2024-11-14T14:00:55,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855343 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855350 2024-11-14T14:00:55,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=553, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855350) roll requested 2024-11-14T14:00:55,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855357 2024-11-14T14:00:55,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855357 2024-11-14T14:00:55,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=554, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855357) roll requested 2024-11-14T14:00:55,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855364 2024-11-14T14:00:55,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855364 2024-11-14T14:00:55,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=555, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855364) roll requested 2024-11-14T14:00:55,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855371 2024-11-14T14:00:55,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855371 2024-11-14T14:00:55,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=556, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855371) roll requested 2024-11-14T14:00:55,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855380 2024-11-14T14:00:55,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855380 2024-11-14T14:00:55,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,386 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=557, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,386 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855380) roll requested 2024-11-14T14:00:55,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855386 2024-11-14T14:00:55,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855386 2024-11-14T14:00:55,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,393 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,393 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=558, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,393 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855386) roll requested 2024-11-14T14:00:55,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855393 2024-11-14T14:00:55,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855393 2024-11-14T14:00:55,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=559, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855393) roll requested 2024-11-14T14:00:55,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855400 2024-11-14T14:00:55,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855393 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855400 2024-11-14T14:00:55,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,407 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=560, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855400) roll requested 2024-11-14T14:00:55,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855407 2024-11-14T14:00:55,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855407 2024-11-14T14:00:55,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=561, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855407) roll requested 2024-11-14T14:00:55,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855415 2024-11-14T14:00:55,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855415 2024-11-14T14:00:55,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,421 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,421 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=562, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,421 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855415) roll requested 2024-11-14T14:00:55,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855421 2024-11-14T14:00:55,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855421 2024-11-14T14:00:55,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,429 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,429 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=563, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,429 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855421) roll requested 2024-11-14T14:00:55,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855429 2024-11-14T14:00:55,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855429 2024-11-14T14:00:55,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=564, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855429) roll requested 2024-11-14T14:00:55,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855436 2024-11-14T14:00:55,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855436 2024-11-14T14:00:55,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=565, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855436) roll requested 2024-11-14T14:00:55,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855443 2024-11-14T14:00:55,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855443 2024-11-14T14:00:55,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=566, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,449 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855443) roll requested 2024-11-14T14:00:55,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855449 2024-11-14T14:00:55,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855449 2024-11-14T14:00:55,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=567, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855449) roll requested 2024-11-14T14:00:55,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855461 2024-11-14T14:00:55,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855449 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855461 2024-11-14T14:00:55,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,468 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=568, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,468 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855461) roll requested 2024-11-14T14:00:55,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855468 2024-11-14T14:00:55,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855468 2024-11-14T14:00:55,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,475 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=569, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855468) roll requested 2024-11-14T14:00:55,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855475 2024-11-14T14:00:55,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855468 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855475 2024-11-14T14:00:55,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,482 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,482 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=570, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,482 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855475) roll requested 2024-11-14T14:00:55,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855482 2024-11-14T14:00:55,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855482 2024-11-14T14:00:55,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=571, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,489 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855482) roll requested 2024-11-14T14:00:55,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855489 2024-11-14T14:00:55,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855489 2024-11-14T14:00:55,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=572, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855489) roll requested 2024-11-14T14:00:55,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855497 2024-11-14T14:00:55,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855497 2024-11-14T14:00:55,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=573, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855497) roll requested 2024-11-14T14:00:55,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855504 2024-11-14T14:00:55,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855504 2024-11-14T14:00:55,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=574, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855504) roll requested 2024-11-14T14:00:55,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855511 2024-11-14T14:00:55,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855511 2024-11-14T14:00:55,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=575, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855511) roll requested 2024-11-14T14:00:55,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855518 2024-11-14T14:00:55,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855518 2024-11-14T14:00:55,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,525 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,525 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=576, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,526 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855518) roll requested 2024-11-14T14:00:55,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855526 2024-11-14T14:00:55,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855526 2024-11-14T14:00:55,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,532 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=577, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855526) roll requested 2024-11-14T14:00:55,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855532 2024-11-14T14:00:55,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855526 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855532 2024-11-14T14:00:55,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,539 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,539 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=578, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,539 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855532) roll requested 2024-11-14T14:00:55,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855539 2024-11-14T14:00:55,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855539 2024-11-14T14:00:55,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,546 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=579, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,546 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855539) roll requested 2024-11-14T14:00:55,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855546 2024-11-14T14:00:55,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855539 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855546 2024-11-14T14:00:55,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,554 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=580, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,554 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855546) roll requested 2024-11-14T14:00:55,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855554 2024-11-14T14:00:55,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855546 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855554 2024-11-14T14:00:55,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=581, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855554) roll requested 2024-11-14T14:00:55,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855565 2024-11-14T14:00:55,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855554 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855565 2024-11-14T14:00:55,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,574 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,574 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=582, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,574 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855565) roll requested 2024-11-14T14:00:55,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855574 2024-11-14T14:00:55,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855574 2024-11-14T14:00:55,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,587 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=583, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855574) roll requested 2024-11-14T14:00:55,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855587 2024-11-14T14:00:55,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855574 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855587 2024-11-14T14:00:55,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=584, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855587) roll requested 2024-11-14T14:00:55,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855596 2024-11-14T14:00:55,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855596 2024-11-14T14:00:55,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=585, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855596) roll requested 2024-11-14T14:00:55,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855604 2024-11-14T14:00:55,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855604 2024-11-14T14:00:55,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,613 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,613 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=586, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,613 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855604) roll requested 2024-11-14T14:00:55,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855613 2024-11-14T14:00:55,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855613 2024-11-14T14:00:55,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=587, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855613) roll requested 2024-11-14T14:00:55,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855625 2024-11-14T14:00:55,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855625 2024-11-14T14:00:55,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=588, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855625) roll requested 2024-11-14T14:00:55,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855633 2024-11-14T14:00:55,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855633 2024-11-14T14:00:55,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=589, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855633) roll requested 2024-11-14T14:00:55,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855640 2024-11-14T14:00:55,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855640 2024-11-14T14:00:55,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,647 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=590, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855640) roll requested 2024-11-14T14:00:55,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855648 2024-11-14T14:00:55,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855648 2024-11-14T14:00:55,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=591, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855648) roll requested 2024-11-14T14:00:55,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855657 2024-11-14T14:00:55,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855657 2024-11-14T14:00:55,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=592, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855657) roll requested 2024-11-14T14:00:55,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855664 2024-11-14T14:00:55,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855664 2024-11-14T14:00:55,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=593, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855664) roll requested 2024-11-14T14:00:55,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855671 2024-11-14T14:00:55,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855671 2024-11-14T14:00:55,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,679 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,679 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=594, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,679 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855671) roll requested 2024-11-14T14:00:55,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855679 2024-11-14T14:00:55,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855679 2024-11-14T14:00:55,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,686 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,686 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=595, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,686 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855679) roll requested 2024-11-14T14:00:55,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855686 2024-11-14T14:00:55,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855679 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855686 2024-11-14T14:00:55,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=596, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855686) roll requested 2024-11-14T14:00:55,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855693 2024-11-14T14:00:55,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855686 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855693 2024-11-14T14:00:55,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=597, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855693) roll requested 2024-11-14T14:00:55,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855700 2024-11-14T14:00:55,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855700 2024-11-14T14:00:55,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,707 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=598, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,707 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855700) roll requested 2024-11-14T14:00:55,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855707 2024-11-14T14:00:55,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855700 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855707 2024-11-14T14:00:55,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=599, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855707) roll requested 2024-11-14T14:00:55,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855716 2024-11-14T14:00:55,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855716 2024-11-14T14:00:55,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,722 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,722 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=600, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,722 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855716) roll requested 2024-11-14T14:00:55,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855722 2024-11-14T14:00:55,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855722 2024-11-14T14:00:55,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,729 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=601, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,729 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855722) roll requested 2024-11-14T14:00:55,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855729 2024-11-14T14:00:55,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855722 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855729 2024-11-14T14:00:55,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,736 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=602, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,736 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855729) roll requested 2024-11-14T14:00:55,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855736 2024-11-14T14:00:55,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855736 2024-11-14T14:00:55,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,743 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=603, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,743 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855736) roll requested 2024-11-14T14:00:55,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855743 2024-11-14T14:00:55,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855736 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855743 2024-11-14T14:00:55,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=604, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855743) roll requested 2024-11-14T14:00:55,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855750 2024-11-14T14:00:55,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855750 2024-11-14T14:00:55,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=605, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855750) roll requested 2024-11-14T14:00:55,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855757 2024-11-14T14:00:55,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855750 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855757 2024-11-14T14:00:55,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,763 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=606, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855757) roll requested 2024-11-14T14:00:55,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855763 2024-11-14T14:00:55,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855763 2024-11-14T14:00:55,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,770 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,770 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=607, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,770 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855763) roll requested 2024-11-14T14:00:55,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855770 2024-11-14T14:00:55,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855763 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855770 2024-11-14T14:00:55,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=608, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855770) roll requested 2024-11-14T14:00:55,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855777 2024-11-14T14:00:55,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855770 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855777 2024-11-14T14:00:55,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=609, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855777) roll requested 2024-11-14T14:00:55,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855785 2024-11-14T14:00:55,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855785 2024-11-14T14:00:55,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=610, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855785) roll requested 2024-11-14T14:00:55,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855792 2024-11-14T14:00:55,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855792 2024-11-14T14:00:55,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=611, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855792) roll requested 2024-11-14T14:00:55,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855799 2024-11-14T14:00:55,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855799 2024-11-14T14:00:55,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=612, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855799) roll requested 2024-11-14T14:00:55,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855806 2024-11-14T14:00:55,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855806 2024-11-14T14:00:55,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=613, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855806) roll requested 2024-11-14T14:00:55,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855814 2024-11-14T14:00:55,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855814 2024-11-14T14:00:55,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=614, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855814) roll requested 2024-11-14T14:00:55,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855821 2024-11-14T14:00:55,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855814 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855821 2024-11-14T14:00:55,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=615, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855821) roll requested 2024-11-14T14:00:55,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855828 2024-11-14T14:00:55,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855828 2024-11-14T14:00:55,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=616, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855828) roll requested 2024-11-14T14:00:55,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855835 2024-11-14T14:00:55,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855835 2024-11-14T14:00:55,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,843 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,843 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=617, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,843 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855835) roll requested 2024-11-14T14:00:55,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855843 2024-11-14T14:00:55,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855843 2024-11-14T14:00:55,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=618, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855843) roll requested 2024-11-14T14:00:55,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855850 2024-11-14T14:00:55,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855843 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855850 2024-11-14T14:00:55,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,857 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=619, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855850) roll requested 2024-11-14T14:00:55,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855857 2024-11-14T14:00:55,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855857 2024-11-14T14:00:55,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=620, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855857) roll requested 2024-11-14T14:00:55,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855865 2024-11-14T14:00:55,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855865 2024-11-14T14:00:55,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=621, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855865) roll requested 2024-11-14T14:00:55,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855871 2024-11-14T14:00:55,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855871 2024-11-14T14:00:55,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=622, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855871) roll requested 2024-11-14T14:00:55,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855879 2024-11-14T14:00:55,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855879 2024-11-14T14:00:55,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,885 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,885 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=623, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,885 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855879) roll requested 2024-11-14T14:00:55,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855885 2024-11-14T14:00:55,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855885 2024-11-14T14:00:55,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,892 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=624, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,892 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855885) roll requested 2024-11-14T14:00:55,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855892 2024-11-14T14:00:55,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855892 2024-11-14T14:00:55,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,898 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,899 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=625, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,899 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855892) roll requested 2024-11-14T14:00:55,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855899 2024-11-14T14:00:55,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855892 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855899 2024-11-14T14:00:55,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=626, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855899) roll requested 2024-11-14T14:00:55,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855905 2024-11-14T14:00:55,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855899 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855905 2024-11-14T14:00:55,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=627, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855905) roll requested 2024-11-14T14:00:55,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855912 2024-11-14T14:00:55,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855912 2024-11-14T14:00:55,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=628, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855912) roll requested 2024-11-14T14:00:55,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855921 2024-11-14T14:00:55,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855921 2024-11-14T14:00:55,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,928 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,928 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=629, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855921) roll requested 2024-11-14T14:00:55,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855928 2024-11-14T14:00:55,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855928 2024-11-14T14:00:55,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,935 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,935 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=630, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,935 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855928) roll requested 2024-11-14T14:00:55,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855935 2024-11-14T14:00:55,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855935 2024-11-14T14:00:55,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=631, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,942 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855935) roll requested 2024-11-14T14:00:55,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855942 2024-11-14T14:00:55,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855935 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855942 2024-11-14T14:00:55,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=632, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855942) roll requested 2024-11-14T14:00:55,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855953 2024-11-14T14:00:55,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855942 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855953 2024-11-14T14:00:55,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=633, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855953) roll requested 2024-11-14T14:00:55,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855960 2024-11-14T14:00:55,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855960 2024-11-14T14:00:55,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=634, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855960) roll requested 2024-11-14T14:00:55,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855967 2024-11-14T14:00:55,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855967 2024-11-14T14:00:55,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=635, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855967) roll requested 2024-11-14T14:00:55,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855973 2024-11-14T14:00:55,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855973 2024-11-14T14:00:55,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,980 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,980 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=636, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,980 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855973) roll requested 2024-11-14T14:00:55,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855980 2024-11-14T14:00:55,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855980 2024-11-14T14:00:55,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=637, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,988 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855980) roll requested 2024-11-14T14:00:55,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855988 2024-11-14T14:00:55,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:55,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855980 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855988 2024-11-14T14:00:55,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:55,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:55,994 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=638, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:55,994 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:55,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855988) roll requested 2024-11-14T14:00:55,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592855994 2024-11-14T14:00:56,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855988 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855994 2024-11-14T14:00:56,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=639, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592855994) roll requested 2024-11-14T14:00:56,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856002 2024-11-14T14:00:56,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592855994 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856002 2024-11-14T14:00:56,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,008 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=640, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856002) roll requested 2024-11-14T14:00:56,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856009 2024-11-14T14:00:56,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856009 2024-11-14T14:00:56,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=641, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856009) roll requested 2024-11-14T14:00:56,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856020 2024-11-14T14:00:56,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856020 2024-11-14T14:00:56,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,027 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,027 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=642, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,027 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856020) roll requested 2024-11-14T14:00:56,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856027 2024-11-14T14:00:56,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856027 2024-11-14T14:00:56,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=643, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856027) roll requested 2024-11-14T14:00:56,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856034 2024-11-14T14:00:56,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856027 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856034 2024-11-14T14:00:56,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=644, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856034) roll requested 2024-11-14T14:00:56,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856041 2024-11-14T14:00:56,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856041 2024-11-14T14:00:56,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=645, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856041) roll requested 2024-11-14T14:00:56,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856049 2024-11-14T14:00:56,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856049 2024-11-14T14:00:56,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=646, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856049) roll requested 2024-11-14T14:00:56,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856055 2024-11-14T14:00:56,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856055 2024-11-14T14:00:56,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=647, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,062 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856055) roll requested 2024-11-14T14:00:56,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856062 2024-11-14T14:00:56,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856062 2024-11-14T14:00:56,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=648, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856062) roll requested 2024-11-14T14:00:56,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856070 2024-11-14T14:00:56,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856062 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856070 2024-11-14T14:00:56,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=649, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856070) roll requested 2024-11-14T14:00:56,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856078 2024-11-14T14:00:56,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856078 2024-11-14T14:00:56,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=650, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856078) roll requested 2024-11-14T14:00:56,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856085 2024-11-14T14:00:56,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856085 2024-11-14T14:00:56,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,093 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=651, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856085) roll requested 2024-11-14T14:00:56,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856093 2024-11-14T14:00:56,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856093 2024-11-14T14:00:56,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,100 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,100 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=652, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,100 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856093) roll requested 2024-11-14T14:00:56,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856100 2024-11-14T14:00:56,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856100 2024-11-14T14:00:56,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,107 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=653, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856100) roll requested 2024-11-14T14:00:56,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856107 2024-11-14T14:00:56,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856100 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856107 2024-11-14T14:00:56,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,114 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=654, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,114 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856107) roll requested 2024-11-14T14:00:56,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856114 2024-11-14T14:00:56,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856114 2024-11-14T14:00:56,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=655, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856114) roll requested 2024-11-14T14:00:56,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856121 2024-11-14T14:00:56,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856114 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856121 2024-11-14T14:00:56,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=656, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856121) roll requested 2024-11-14T14:00:56,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856129 2024-11-14T14:00:56,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856129 2024-11-14T14:00:56,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=657, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856129) roll requested 2024-11-14T14:00:56,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856136 2024-11-14T14:00:56,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856136 2024-11-14T14:00:56,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,142 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=658, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,143 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856136) roll requested 2024-11-14T14:00:56,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856143 2024-11-14T14:00:56,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856143 2024-11-14T14:00:56,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=659, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856143) roll requested 2024-11-14T14:00:56,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856149 2024-11-14T14:00:56,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856143 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856149 2024-11-14T14:00:56,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,157 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=660, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856149) roll requested 2024-11-14T14:00:56,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856157 2024-11-14T14:00:56,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856157 2024-11-14T14:00:56,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=661, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856157) roll requested 2024-11-14T14:00:56,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856165 2024-11-14T14:00:56,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856165 2024-11-14T14:00:56,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=662, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856165) roll requested 2024-11-14T14:00:56,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856177 2024-11-14T14:00:56,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856177 2024-11-14T14:00:56,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=663, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856177) roll requested 2024-11-14T14:00:56,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856185 2024-11-14T14:00:56,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856185 2024-11-14T14:00:56,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=664, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856185) roll requested 2024-11-14T14:00:56,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856197 2024-11-14T14:00:56,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856197 2024-11-14T14:00:56,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=665, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856197) roll requested 2024-11-14T14:00:56,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856209 2024-11-14T14:00:56,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856209 2024-11-14T14:00:56,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=666, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856209) roll requested 2024-11-14T14:00:56,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856217 2024-11-14T14:00:56,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856217 2024-11-14T14:00:56,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=667, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856217) roll requested 2024-11-14T14:00:56,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856225 2024-11-14T14:00:56,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856225 2024-11-14T14:00:56,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=668, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856225) roll requested 2024-11-14T14:00:56,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856232 2024-11-14T14:00:56,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856232 2024-11-14T14:00:56,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,242 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=669, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,242 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856232) roll requested 2024-11-14T14:00:56,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856242 2024-11-14T14:00:56,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856242 2024-11-14T14:00:56,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,249 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,249 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=670, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,249 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856242) roll requested 2024-11-14T14:00:56,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856249 2024-11-14T14:00:56,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856242 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856249 2024-11-14T14:00:56,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=671, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856249) roll requested 2024-11-14T14:00:56,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856257 2024-11-14T14:00:56,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856249 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856257 2024-11-14T14:00:56,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=672, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856257) roll requested 2024-11-14T14:00:56,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856264 2024-11-14T14:00:56,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856264 2024-11-14T14:00:56,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=673, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856264) roll requested 2024-11-14T14:00:56,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856271 2024-11-14T14:00:56,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856271 2024-11-14T14:00:56,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,278 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=674, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,278 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856271) roll requested 2024-11-14T14:00:56,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856278 2024-11-14T14:00:56,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856278 2024-11-14T14:00:56,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=675, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856278) roll requested 2024-11-14T14:00:56,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856285 2024-11-14T14:00:56,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856285 2024-11-14T14:00:56,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=676, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856285) roll requested 2024-11-14T14:00:56,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856292 2024-11-14T14:00:56,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856292 2024-11-14T14:00:56,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,299 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=677, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856292) roll requested 2024-11-14T14:00:56,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856299 2024-11-14T14:00:56,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856299 2024-11-14T14:00:56,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,307 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=678, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,307 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856299) roll requested 2024-11-14T14:00:56,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856307 2024-11-14T14:00:56,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856307 2024-11-14T14:00:56,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,313 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=679, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856307) roll requested 2024-11-14T14:00:56,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856314 2024-11-14T14:00:56,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856314 2024-11-14T14:00:56,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=680, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856314) roll requested 2024-11-14T14:00:56,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856321 2024-11-14T14:00:56,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856321 2024-11-14T14:00:56,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=681, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856321) roll requested 2024-11-14T14:00:56,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856328 2024-11-14T14:00:56,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856328 2024-11-14T14:00:56,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,335 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,335 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=682, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856328) roll requested 2024-11-14T14:00:56,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856335 2024-11-14T14:00:56,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856328 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856335 2024-11-14T14:00:56,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=683, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856335) roll requested 2024-11-14T14:00:56,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856342 2024-11-14T14:00:56,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856342 2024-11-14T14:00:56,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=684, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856342) roll requested 2024-11-14T14:00:56,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856349 2024-11-14T14:00:56,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856349 2024-11-14T14:00:56,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=685, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856349) roll requested 2024-11-14T14:00:56,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856361 2024-11-14T14:00:56,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856361 2024-11-14T14:00:56,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=686, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856361) roll requested 2024-11-14T14:00:56,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856368 2024-11-14T14:00:56,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856368 2024-11-14T14:00:56,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,375 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=687, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856368) roll requested 2024-11-14T14:00:56,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856375 2024-11-14T14:00:56,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856375 2024-11-14T14:00:56,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,382 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,382 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=688, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,382 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856375) roll requested 2024-11-14T14:00:56,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856383 2024-11-14T14:00:56,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856383 2024-11-14T14:00:56,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=689, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856383) roll requested 2024-11-14T14:00:56,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856389 2024-11-14T14:00:56,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856383 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856389 2024-11-14T14:00:56,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=690, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856389) roll requested 2024-11-14T14:00:56,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856397 2024-11-14T14:00:56,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856397 2024-11-14T14:00:56,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,403 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=691, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,403 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856397) roll requested 2024-11-14T14:00:56,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856403 2024-11-14T14:00:56,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856403 2024-11-14T14:00:56,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,410 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,410 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=692, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,410 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856403) roll requested 2024-11-14T14:00:56,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856410 2024-11-14T14:00:56,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856403 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856410 2024-11-14T14:00:56,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=693, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856410) roll requested 2024-11-14T14:00:56,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856417 2024-11-14T14:00:56,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856410 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856417 2024-11-14T14:00:56,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=694, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856417) roll requested 2024-11-14T14:00:56,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856424 2024-11-14T14:00:56,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856424 2024-11-14T14:00:56,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=695, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856424) roll requested 2024-11-14T14:00:56,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856431 2024-11-14T14:00:56,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856431 2024-11-14T14:00:56,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,438 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,438 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=696, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856431) roll requested 2024-11-14T14:00:56,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856438 2024-11-14T14:00:56,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856438 2024-11-14T14:00:56,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,445 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=697, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856438) roll requested 2024-11-14T14:00:56,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856446 2024-11-14T14:00:56,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856446 2024-11-14T14:00:56,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,456 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=698, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,456 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856446) roll requested 2024-11-14T14:00:56,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856456 2024-11-14T14:00:56,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856456 2024-11-14T14:00:56,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,463 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=699, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,463 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856456) roll requested 2024-11-14T14:00:56,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856463 2024-11-14T14:00:56,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856463 2024-11-14T14:00:56,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,470 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=700, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,470 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856463) roll requested 2024-11-14T14:00:56,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856470 2024-11-14T14:00:56,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856463 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856470 2024-11-14T14:00:56,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=701, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856470) roll requested 2024-11-14T14:00:56,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856477 2024-11-14T14:00:56,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856477 2024-11-14T14:00:56,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=702, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856477) roll requested 2024-11-14T14:00:56,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856484 2024-11-14T14:00:56,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856484 2024-11-14T14:00:56,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=703, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,491 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856484) roll requested 2024-11-14T14:00:56,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856491 2024-11-14T14:00:56,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856491 2024-11-14T14:00:56,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=704, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856491) roll requested 2024-11-14T14:00:56,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856499 2024-11-14T14:00:56,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856499 2024-11-14T14:00:56,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,506 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,506 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=705, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,506 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856499) roll requested 2024-11-14T14:00:56,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856506 2024-11-14T14:00:56,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856506 2024-11-14T14:00:56,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,513 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,513 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=706, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,514 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856506) roll requested 2024-11-14T14:00:56,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856514 2024-11-14T14:00:56,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856506 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856514 2024-11-14T14:00:56,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,521 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=707, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856514) roll requested 2024-11-14T14:00:56,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856521 2024-11-14T14:00:56,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856514 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856521 2024-11-14T14:00:56,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=708, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856521) roll requested 2024-11-14T14:00:56,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856528 2024-11-14T14:00:56,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856528 2024-11-14T14:00:56,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=709, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856528) roll requested 2024-11-14T14:00:56,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856535 2024-11-14T14:00:56,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856535 2024-11-14T14:00:56,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=710, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856535) roll requested 2024-11-14T14:00:56,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856542 2024-11-14T14:00:56,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856542 2024-11-14T14:00:56,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,549 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=711, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,549 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856542) roll requested 2024-11-14T14:00:56,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856550 2024-11-14T14:00:56,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856550 2024-11-14T14:00:56,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=712, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856550) roll requested 2024-11-14T14:00:56,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856557 2024-11-14T14:00:56,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856557 2024-11-14T14:00:56,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=713, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856557) roll requested 2024-11-14T14:00:56,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856564 2024-11-14T14:00:56,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856564 2024-11-14T14:00:56,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,570 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=714, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,570 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856564) roll requested 2024-11-14T14:00:56,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856570 2024-11-14T14:00:56,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856570 2024-11-14T14:00:56,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=715, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856570) roll requested 2024-11-14T14:00:56,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856577 2024-11-14T14:00:56,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856570 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856577 2024-11-14T14:00:56,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,583 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=716, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856577) roll requested 2024-11-14T14:00:56,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856584 2024-11-14T14:00:56,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856584 2024-11-14T14:00:56,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,590 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,591 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=717, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,591 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856584) roll requested 2024-11-14T14:00:56,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856591 2024-11-14T14:00:56,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856591 2024-11-14T14:00:56,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,598 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=718, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,598 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856591) roll requested 2024-11-14T14:00:56,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856598 2024-11-14T14:00:56,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856591 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856598 2024-11-14T14:00:56,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,605 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=719, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,605 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856598) roll requested 2024-11-14T14:00:56,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856605 2024-11-14T14:00:56,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856598 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856605 2024-11-14T14:00:56,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=720, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856605) roll requested 2024-11-14T14:00:56,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856612 2024-11-14T14:00:56,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856605 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856612 2024-11-14T14:00:56,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=721, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856612) roll requested 2024-11-14T14:00:56,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856618 2024-11-14T14:00:56,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856612 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856618 2024-11-14T14:00:56,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=722, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856618) roll requested 2024-11-14T14:00:56,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856625 2024-11-14T14:00:56,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856625 2024-11-14T14:00:56,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,631 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,631 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=723, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,631 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856625) roll requested 2024-11-14T14:00:56,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856631 2024-11-14T14:00:56,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856631 2024-11-14T14:00:56,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,638 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=724, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856631) roll requested 2024-11-14T14:00:56,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856638 2024-11-14T14:00:56,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856631 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856638 2024-11-14T14:00:56,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=725, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856638) roll requested 2024-11-14T14:00:56,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856645 2024-11-14T14:00:56,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856645 2024-11-14T14:00:56,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=726, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856645) roll requested 2024-11-14T14:00:56,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856652 2024-11-14T14:00:56,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856652 2024-11-14T14:00:56,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,660 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=727, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856652) roll requested 2024-11-14T14:00:56,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856660 2024-11-14T14:00:56,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856660 2024-11-14T14:00:56,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,667 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=728, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856660) roll requested 2024-11-14T14:00:56,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856667 2024-11-14T14:00:56,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856667 2024-11-14T14:00:56,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=729, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856667) roll requested 2024-11-14T14:00:56,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856674 2024-11-14T14:00:56,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856674 2024-11-14T14:00:56,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=730, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856674) roll requested 2024-11-14T14:00:56,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856681 2024-11-14T14:00:56,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856681 2024-11-14T14:00:56,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,688 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=731, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856681) roll requested 2024-11-14T14:00:56,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856689 2024-11-14T14:00:56,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856689 2024-11-14T14:00:56,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=732, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856689) roll requested 2024-11-14T14:00:56,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856695 2024-11-14T14:00:56,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856695 2024-11-14T14:00:56,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=733, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856695) roll requested 2024-11-14T14:00:56,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856702 2024-11-14T14:00:56,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856702 2024-11-14T14:00:56,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=734, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856702) roll requested 2024-11-14T14:00:56,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856709 2024-11-14T14:00:56,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856709 2024-11-14T14:00:56,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=735, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856709) roll requested 2024-11-14T14:00:56,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856717 2024-11-14T14:00:56,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856717 2024-11-14T14:00:56,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=736, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856717) roll requested 2024-11-14T14:00:56,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856724 2024-11-14T14:00:56,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856724 2024-11-14T14:00:56,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,731 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,731 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=737, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,731 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856724) roll requested 2024-11-14T14:00:56,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856731 2024-11-14T14:00:56,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856731 2024-11-14T14:00:56,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,738 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,738 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=738, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,738 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856731) roll requested 2024-11-14T14:00:56,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856738 2024-11-14T14:00:56,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856731 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856738 2024-11-14T14:00:56,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=739, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856738) roll requested 2024-11-14T14:00:56,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856745 2024-11-14T14:00:56,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856738 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856745 2024-11-14T14:00:56,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=740, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856745) roll requested 2024-11-14T14:00:56,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856753 2024-11-14T14:00:56,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856753 2024-11-14T14:00:56,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=741, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856753) roll requested 2024-11-14T14:00:56,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856759 2024-11-14T14:00:56,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856759 2024-11-14T14:00:56,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=742, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856759) roll requested 2024-11-14T14:00:56,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856767 2024-11-14T14:00:56,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856767 2024-11-14T14:00:56,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,775 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=743, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856767) roll requested 2024-11-14T14:00:56,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856776 2024-11-14T14:00:56,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856776 2024-11-14T14:00:56,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,783 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=744, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,783 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856776) roll requested 2024-11-14T14:00:56,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856784 2024-11-14T14:00:56,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856784 2024-11-14T14:00:56,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=745, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856784) roll requested 2024-11-14T14:00:56,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856792 2024-11-14T14:00:56,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856792 2024-11-14T14:00:56,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=746, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856792) roll requested 2024-11-14T14:00:56,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856799 2024-11-14T14:00:56,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856799 2024-11-14T14:00:56,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,807 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=747, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,807 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856799) roll requested 2024-11-14T14:00:56,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856807 2024-11-14T14:00:56,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856807 2024-11-14T14:00:56,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=748, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,814 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856807) roll requested 2024-11-14T14:00:56,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856814 2024-11-14T14:00:56,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856807 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856814 2024-11-14T14:00:56,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=749, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856814) roll requested 2024-11-14T14:00:56,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856821 2024-11-14T14:00:56,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856814 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856821 2024-11-14T14:00:56,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=750, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856821) roll requested 2024-11-14T14:00:56,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856827 2024-11-14T14:00:56,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856827 2024-11-14T14:00:56,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,834 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=751, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856827) roll requested 2024-11-14T14:00:56,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856835 2024-11-14T14:00:56,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856827 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856835 2024-11-14T14:00:56,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=752, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856835) roll requested 2024-11-14T14:00:56,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856842 2024-11-14T14:00:56,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856842 2024-11-14T14:00:56,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=753, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856842) roll requested 2024-11-14T14:00:56,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856849 2024-11-14T14:00:56,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856849 2024-11-14T14:00:56,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=754, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856849) roll requested 2024-11-14T14:00:56,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856856 2024-11-14T14:00:56,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856856 2024-11-14T14:00:56,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,862 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,863 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=755, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,863 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856856) roll requested 2024-11-14T14:00:56,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856863 2024-11-14T14:00:56,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856863 2024-11-14T14:00:56,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,869 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,869 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=756, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,869 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856863) roll requested 2024-11-14T14:00:56,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856869 2024-11-14T14:00:56,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856863 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856869 2024-11-14T14:00:56,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,876 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,877 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=757, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,877 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856869) roll requested 2024-11-14T14:00:56,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856877 2024-11-14T14:00:56,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856869 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856877 2024-11-14T14:00:56,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=758, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856877) roll requested 2024-11-14T14:00:56,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856884 2024-11-14T14:00:56,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856877 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856884 2024-11-14T14:00:56,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,891 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,891 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=759, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,891 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856884) roll requested 2024-11-14T14:00:56,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856892 2024-11-14T14:00:56,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856892 2024-11-14T14:00:56,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,898 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,898 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=760, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,898 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856892) roll requested 2024-11-14T14:00:56,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856898 2024-11-14T14:00:56,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856892 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856898 2024-11-14T14:00:56,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=761, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856898) roll requested 2024-11-14T14:00:56,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856905 2024-11-14T14:00:56,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856898 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856905 2024-11-14T14:00:56,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=762, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856905) roll requested 2024-11-14T14:00:56,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856912 2024-11-14T14:00:56,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856912 2024-11-14T14:00:56,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=763, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,919 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856912) roll requested 2024-11-14T14:00:56,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856919 2024-11-14T14:00:56,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856919 2024-11-14T14:00:56,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=764, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856919) roll requested 2024-11-14T14:00:56,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856926 2024-11-14T14:00:56,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856919 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856926 2024-11-14T14:00:56,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=765, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856926) roll requested 2024-11-14T14:00:56,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856933 2024-11-14T14:00:56,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856926 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856933 2024-11-14T14:00:56,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=766, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856933) roll requested 2024-11-14T14:00:56,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856940 2024-11-14T14:00:56,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856940 2024-11-14T14:00:56,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,947 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=767, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,947 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856940) roll requested 2024-11-14T14:00:56,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856947 2024-11-14T14:00:56,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856947 2024-11-14T14:00:56,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,954 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=768, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856947) roll requested 2024-11-14T14:00:56,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856954 2024-11-14T14:00:56,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856947 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856954 2024-11-14T14:00:56,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,961 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=769, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,961 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856954) roll requested 2024-11-14T14:00:56,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856961 2024-11-14T14:00:56,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856961 2024-11-14T14:00:56,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,968 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,968 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=770, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,968 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856961) roll requested 2024-11-14T14:00:56,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856968 2024-11-14T14:00:56,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856968 2024-11-14T14:00:56,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=771, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,975 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856968) roll requested 2024-11-14T14:00:56,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856975 2024-11-14T14:00:56,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856975 2024-11-14T14:00:56,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,982 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,982 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=772, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,982 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856975) roll requested 2024-11-14T14:00:56,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856982 2024-11-14T14:00:56,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856982 2024-11-14T14:00:56,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=773, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856982) roll requested 2024-11-14T14:00:56,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856989 2024-11-14T14:00:56,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:56,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856982 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856989 2024-11-14T14:00:56,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:56,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:56,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=774, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:56,996 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:56,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856989) roll requested 2024-11-14T14:00:56,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592856996 2024-11-14T14:00:57,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856996 2024-11-14T14:00:57,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=775, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592856996) roll requested 2024-11-14T14:00:57,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857003 2024-11-14T14:00:57,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592856996 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857003 2024-11-14T14:00:57,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,010 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=776, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857003) roll requested 2024-11-14T14:00:57,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857010 2024-11-14T14:00:57,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857010 2024-11-14T14:00:57,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,017 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=777, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857010) roll requested 2024-11-14T14:00:57,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857018 2024-11-14T14:00:57,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857018 2024-11-14T14:00:57,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=778, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857018) roll requested 2024-11-14T14:00:57,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857025 2024-11-14T14:00:57,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857025 2024-11-14T14:00:57,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=779, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857025) roll requested 2024-11-14T14:00:57,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857034 2024-11-14T14:00:57,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857034 2024-11-14T14:00:57,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=780, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857034) roll requested 2024-11-14T14:00:57,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857045 2024-11-14T14:00:57,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857045 2024-11-14T14:00:57,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=781, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857045) roll requested 2024-11-14T14:00:57,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857055 2024-11-14T14:00:57,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857055 2024-11-14T14:00:57,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,063 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,063 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=782, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857055) roll requested 2024-11-14T14:00:57,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857063 2024-11-14T14:00:57,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857063 2024-11-14T14:00:57,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,071 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=783, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,071 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857063) roll requested 2024-11-14T14:00:57,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857071 2024-11-14T14:00:57,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857071 2024-11-14T14:00:57,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=784, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857071) roll requested 2024-11-14T14:00:57,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857078 2024-11-14T14:00:57,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857078 2024-11-14T14:00:57,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=785, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857078) roll requested 2024-11-14T14:00:57,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857084 2024-11-14T14:00:57,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857084 2024-11-14T14:00:57,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=786, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857084) roll requested 2024-11-14T14:00:57,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857091 2024-11-14T14:00:57,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857091 2024-11-14T14:00:57,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=787, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857091) roll requested 2024-11-14T14:00:57,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857098 2024-11-14T14:00:57,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857098 2024-11-14T14:00:57,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=788, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857098) roll requested 2024-11-14T14:00:57,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857105 2024-11-14T14:00:57,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857105 2024-11-14T14:00:57,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=789, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857105) roll requested 2024-11-14T14:00:57,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857113 2024-11-14T14:00:57,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857113 2024-11-14T14:00:57,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=790, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857113) roll requested 2024-11-14T14:00:57,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857120 2024-11-14T14:00:57,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857120 2024-11-14T14:00:57,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=791, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857120) roll requested 2024-11-14T14:00:57,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857128 2024-11-14T14:00:57,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857128 2024-11-14T14:00:57,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=792, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857128) roll requested 2024-11-14T14:00:57,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857135 2024-11-14T14:00:57,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857128 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857135 2024-11-14T14:00:57,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,142 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=793, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,142 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857135) roll requested 2024-11-14T14:00:57,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857142 2024-11-14T14:00:57,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857142 2024-11-14T14:00:57,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=794, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857142) roll requested 2024-11-14T14:00:57,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857149 2024-11-14T14:00:57,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857142 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857149 2024-11-14T14:00:57,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=795, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857149) roll requested 2024-11-14T14:00:57,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857156 2024-11-14T14:00:57,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857156 2024-11-14T14:00:57,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,163 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,163 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=796, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,163 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857156) roll requested 2024-11-14T14:00:57,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857163 2024-11-14T14:00:57,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857163 2024-11-14T14:00:57,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=797, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857163) roll requested 2024-11-14T14:00:57,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857170 2024-11-14T14:00:57,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857163 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857170 2024-11-14T14:00:57,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=798, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857170) roll requested 2024-11-14T14:00:57,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857177 2024-11-14T14:00:57,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857170 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857177 2024-11-14T14:00:57,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=799, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857177) roll requested 2024-11-14T14:00:57,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857184 2024-11-14T14:00:57,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857184 2024-11-14T14:00:57,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,191 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=800, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,191 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857184) roll requested 2024-11-14T14:00:57,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857191 2024-11-14T14:00:57,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857191 2024-11-14T14:00:57,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=801, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857191) roll requested 2024-11-14T14:00:57,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857198 2024-11-14T14:00:57,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857191 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857198 2024-11-14T14:00:57,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=802, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857198) roll requested 2024-11-14T14:00:57,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857205 2024-11-14T14:00:57,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857205 2024-11-14T14:00:57,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,212 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,212 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=803, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857205) roll requested 2024-11-14T14:00:57,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857212 2024-11-14T14:00:57,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857212 2024-11-14T14:00:57,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=804, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857212) roll requested 2024-11-14T14:00:57,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857219 2024-11-14T14:00:57,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857219 2024-11-14T14:00:57,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=805, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857219) roll requested 2024-11-14T14:00:57,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857226 2024-11-14T14:00:57,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857226 2024-11-14T14:00:57,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=806, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857226) roll requested 2024-11-14T14:00:57,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857233 2024-11-14T14:00:57,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857233 2024-11-14T14:00:57,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,241 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=807, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857233) roll requested 2024-11-14T14:00:57,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857241 2024-11-14T14:00:57,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857241 2024-11-14T14:00:57,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,248 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=808, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857241) roll requested 2024-11-14T14:00:57,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857248 2024-11-14T14:00:57,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857248 2024-11-14T14:00:57,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=809, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857248) roll requested 2024-11-14T14:00:57,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857255 2024-11-14T14:00:57,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857255 2024-11-14T14:00:57,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,262 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,262 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=810, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,262 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857255) roll requested 2024-11-14T14:00:57,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857262 2024-11-14T14:00:57,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857262 2024-11-14T14:00:57,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=811, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857262) roll requested 2024-11-14T14:00:57,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857269 2024-11-14T14:00:57,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857262 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857269 2024-11-14T14:00:57,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,275 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=812, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857269) roll requested 2024-11-14T14:00:57,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857276 2024-11-14T14:00:57,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857276 2024-11-14T14:00:57,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=813, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857276) roll requested 2024-11-14T14:00:57,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857283 2024-11-14T14:00:57,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857276 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857283 2024-11-14T14:00:57,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=814, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,290 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857283) roll requested 2024-11-14T14:00:57,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857290 2024-11-14T14:00:57,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857290 2024-11-14T14:00:57,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=815, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857290) roll requested 2024-11-14T14:00:57,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857297 2024-11-14T14:00:57,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857297 2024-11-14T14:00:57,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=816, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857297) roll requested 2024-11-14T14:00:57,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857303 2024-11-14T14:00:57,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857303 2024-11-14T14:00:57,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,310 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=817, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857303) roll requested 2024-11-14T14:00:57,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857311 2024-11-14T14:00:57,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857311 2024-11-14T14:00:57,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=818, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,319 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857311) roll requested 2024-11-14T14:00:57,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857319 2024-11-14T14:00:57,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857319 2024-11-14T14:00:57,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=819, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857319) roll requested 2024-11-14T14:00:57,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857326 2024-11-14T14:00:57,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857319 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857326 2024-11-14T14:00:57,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=820, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857326) roll requested 2024-11-14T14:00:57,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857333 2024-11-14T14:00:57,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857326 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857333 2024-11-14T14:00:57,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=821, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857333) roll requested 2024-11-14T14:00:57,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857340 2024-11-14T14:00:57,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857340 2024-11-14T14:00:57,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,347 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,347 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=822, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,347 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857340) roll requested 2024-11-14T14:00:57,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857347 2024-11-14T14:00:57,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857347 2024-11-14T14:00:57,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,353 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=823, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857347) roll requested 2024-11-14T14:00:57,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857354 2024-11-14T14:00:57,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857347 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857354 2024-11-14T14:00:57,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=824, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857354) roll requested 2024-11-14T14:00:57,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857361 2024-11-14T14:00:57,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857354 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857361 2024-11-14T14:00:57,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,367 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=825, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857361) roll requested 2024-11-14T14:00:57,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857368 2024-11-14T14:00:57,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857368 2024-11-14T14:00:57,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=826, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857368) roll requested 2024-11-14T14:00:57,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857375 2024-11-14T14:00:57,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857375 2024-11-14T14:00:57,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=827, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857375) roll requested 2024-11-14T14:00:57,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857381 2024-11-14T14:00:57,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857381 2024-11-14T14:00:57,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=828, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857381) roll requested 2024-11-14T14:00:57,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857388 2024-11-14T14:00:57,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857388 2024-11-14T14:00:57,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,395 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=829, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,395 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857388) roll requested 2024-11-14T14:00:57,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857395 2024-11-14T14:00:57,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857388 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857395 2024-11-14T14:00:57,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=830, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857395) roll requested 2024-11-14T14:00:57,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857403 2024-11-14T14:00:57,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857403 2024-11-14T14:00:57,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,409 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,410 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=831, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,410 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857403) roll requested 2024-11-14T14:00:57,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857410 2024-11-14T14:00:57,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857403 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857410 2024-11-14T14:00:57,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=832, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857410) roll requested 2024-11-14T14:00:57,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857417 2024-11-14T14:00:57,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857410 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857417 2024-11-14T14:00:57,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=833, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857417) roll requested 2024-11-14T14:00:57,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857424 2024-11-14T14:00:57,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857424 2024-11-14T14:00:57,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=834, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857424) roll requested 2024-11-14T14:00:57,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857431 2024-11-14T14:00:57,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857431 2024-11-14T14:00:57,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,437 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=835, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,437 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857431) roll requested 2024-11-14T14:00:57,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857437 2024-11-14T14:00:57,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857437 2024-11-14T14:00:57,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,444 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,445 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=836, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857437) roll requested 2024-11-14T14:00:57,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857445 2024-11-14T14:00:57,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857437 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857445 2024-11-14T14:00:57,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,454 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,454 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=837, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857445) roll requested 2024-11-14T14:00:57,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857454 2024-11-14T14:00:57,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857454 2024-11-14T14:00:57,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,462 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=838, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857454) roll requested 2024-11-14T14:00:57,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857462 2024-11-14T14:00:57,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857462 2024-11-14T14:00:57,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=839, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857462) roll requested 2024-11-14T14:00:57,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857469 2024-11-14T14:00:57,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857469 2024-11-14T14:00:57,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,476 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=840, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,476 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857469) roll requested 2024-11-14T14:00:57,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857476 2024-11-14T14:00:57,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857476 2024-11-14T14:00:57,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=841, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857476) roll requested 2024-11-14T14:00:57,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857483 2024-11-14T14:00:57,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857483 2024-11-14T14:00:57,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=842, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857483) roll requested 2024-11-14T14:00:57,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857491 2024-11-14T14:00:57,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857491 2024-11-14T14:00:57,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=843, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857491) roll requested 2024-11-14T14:00:57,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857498 2024-11-14T14:00:57,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857498 2024-11-14T14:00:57,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=844, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857498) roll requested 2024-11-14T14:00:57,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857509 2024-11-14T14:00:57,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857509 2024-11-14T14:00:57,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,516 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,516 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=845, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,516 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857509) roll requested 2024-11-14T14:00:57,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857516 2024-11-14T14:00:57,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857516 2024-11-14T14:00:57,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=846, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857516) roll requested 2024-11-14T14:00:57,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857523 2024-11-14T14:00:57,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857516 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857523 2024-11-14T14:00:57,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=847, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,530 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857523) roll requested 2024-11-14T14:00:57,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857530 2024-11-14T14:00:57,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857530 2024-11-14T14:00:57,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=848, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857530) roll requested 2024-11-14T14:00:57,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857537 2024-11-14T14:00:57,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857537 2024-11-14T14:00:57,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=849, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857537) roll requested 2024-11-14T14:00:57,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857544 2024-11-14T14:00:57,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857544 2024-11-14T14:00:57,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=850, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857544) roll requested 2024-11-14T14:00:57,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857552 2024-11-14T14:00:57,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857552 2024-11-14T14:00:57,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,558 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=851, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857552) roll requested 2024-11-14T14:00:57,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857558 2024-11-14T14:00:57,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857558 2024-11-14T14:00:57,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,565 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,565 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=852, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857558) roll requested 2024-11-14T14:00:57,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857565 2024-11-14T14:00:57,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857565 2024-11-14T14:00:57,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=853, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857565) roll requested 2024-11-14T14:00:57,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857572 2024-11-14T14:00:57,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857572 2024-11-14T14:00:57,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,579 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=854, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857572) roll requested 2024-11-14T14:00:57,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857580 2024-11-14T14:00:57,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857580 2024-11-14T14:00:57,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=855, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,586 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857580) roll requested 2024-11-14T14:00:57,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857587 2024-11-14T14:00:57,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857587 2024-11-14T14:00:57,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=856, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857587) roll requested 2024-11-14T14:00:57,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857593 2024-11-14T14:00:57,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857593 2024-11-14T14:00:57,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=857, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857593) roll requested 2024-11-14T14:00:57,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857601 2024-11-14T14:00:57,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857601 2024-11-14T14:00:57,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=858, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857601) roll requested 2024-11-14T14:00:57,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857608 2024-11-14T14:00:57,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857608 2024-11-14T14:00:57,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,614 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=859, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,614 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857608) roll requested 2024-11-14T14:00:57,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857614 2024-11-14T14:00:57,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857614 2024-11-14T14:00:57,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=860, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857614) roll requested 2024-11-14T14:00:57,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857621 2024-11-14T14:00:57,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857621 2024-11-14T14:00:57,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=861, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857621) roll requested 2024-11-14T14:00:57,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857628 2024-11-14T14:00:57,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857628 2024-11-14T14:00:57,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=862, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857628) roll requested 2024-11-14T14:00:57,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857635 2024-11-14T14:00:57,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857635 2024-11-14T14:00:57,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=863, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857635) roll requested 2024-11-14T14:00:57,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857642 2024-11-14T14:00:57,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857642 2024-11-14T14:00:57,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,649 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,650 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=864, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,650 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857642) roll requested 2024-11-14T14:00:57,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857650 2024-11-14T14:00:57,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857650 2024-11-14T14:00:57,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=865, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857650) roll requested 2024-11-14T14:00:57,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857657 2024-11-14T14:00:57,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857650 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857657 2024-11-14T14:00:57,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=866, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857657) roll requested 2024-11-14T14:00:57,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857664 2024-11-14T14:00:57,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857664 2024-11-14T14:00:57,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=867, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857664) roll requested 2024-11-14T14:00:57,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857671 2024-11-14T14:00:57,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857671 2024-11-14T14:00:57,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=868, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857671) roll requested 2024-11-14T14:00:57,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857678 2024-11-14T14:00:57,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857678 2024-11-14T14:00:57,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,685 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,685 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=869, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,685 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857678) roll requested 2024-11-14T14:00:57,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857685 2024-11-14T14:00:57,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857685 2024-11-14T14:00:57,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,692 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,692 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=870, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,692 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857685) roll requested 2024-11-14T14:00:57,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857692 2024-11-14T14:00:57,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857692 2024-11-14T14:00:57,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,699 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=871, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,699 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857692) roll requested 2024-11-14T14:00:57,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857699 2024-11-14T14:00:57,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857692 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857699 2024-11-14T14:00:57,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,706 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,706 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=872, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,706 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857699) roll requested 2024-11-14T14:00:57,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857706 2024-11-14T14:00:57,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857699 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857706 2024-11-14T14:00:57,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,715 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=873, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,715 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857706) roll requested 2024-11-14T14:00:57,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857715 2024-11-14T14:00:57,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857706 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857715 2024-11-14T14:00:57,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,722 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,722 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=874, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,722 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857715) roll requested 2024-11-14T14:00:57,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857722 2024-11-14T14:00:57,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857715 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857722 2024-11-14T14:00:57,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=875, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857722) roll requested 2024-11-14T14:00:57,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857733 2024-11-14T14:00:57,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857722 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857733 2024-11-14T14:00:57,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=876, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857733) roll requested 2024-11-14T14:00:57,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857745 2024-11-14T14:00:57,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857745 2024-11-14T14:00:57,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,751 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,751 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=877, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,751 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857745) roll requested 2024-11-14T14:00:57,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857751 2024-11-14T14:00:57,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857751 2024-11-14T14:00:57,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=878, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857751) roll requested 2024-11-14T14:00:57,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857759 2024-11-14T14:00:57,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857759 2024-11-14T14:00:57,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=879, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857759) roll requested 2024-11-14T14:00:57,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857766 2024-11-14T14:00:57,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857766 2024-11-14T14:00:57,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,774 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=880, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857766) roll requested 2024-11-14T14:00:57,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857774 2024-11-14T14:00:57,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857774 2024-11-14T14:00:57,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=881, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,781 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857774) roll requested 2024-11-14T14:00:57,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857781 2024-11-14T14:00:57,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857781 2024-11-14T14:00:57,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,789 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=882, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,790 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857781) roll requested 2024-11-14T14:00:57,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857790 2024-11-14T14:00:57,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857781 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857790 2024-11-14T14:00:57,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,797 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,797 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=883, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857790) roll requested 2024-11-14T14:00:57,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857797 2024-11-14T14:00:57,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857790 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857797 2024-11-14T14:00:57,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=884, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857797) roll requested 2024-11-14T14:00:57,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857808 2024-11-14T14:00:57,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857808 2024-11-14T14:00:57,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=885, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857808) roll requested 2024-11-14T14:00:57,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857817 2024-11-14T14:00:57,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857817 2024-11-14T14:00:57,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=886, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857817) roll requested 2024-11-14T14:00:57,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857824 2024-11-14T14:00:57,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857824 2024-11-14T14:00:57,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=887, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857824) roll requested 2024-11-14T14:00:57,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857831 2024-11-14T14:00:57,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857831 2024-11-14T14:00:57,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,838 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=888, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857831) roll requested 2024-11-14T14:00:57,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857839 2024-11-14T14:00:57,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857839 2024-11-14T14:00:57,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=889, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857839) roll requested 2024-11-14T14:00:57,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857846 2024-11-14T14:00:57,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857846 2024-11-14T14:00:57,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=890, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857846) roll requested 2024-11-14T14:00:57,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857853 2024-11-14T14:00:57,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857853 2024-11-14T14:00:57,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=891, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857853) roll requested 2024-11-14T14:00:57,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857860 2024-11-14T14:00:57,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857860 2024-11-14T14:00:57,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,867 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,867 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=892, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,867 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857860) roll requested 2024-11-14T14:00:57,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857867 2024-11-14T14:00:57,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857867 2024-11-14T14:00:57,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,875 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=893, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857867) roll requested 2024-11-14T14:00:57,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857875 2024-11-14T14:00:57,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857867 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857875 2024-11-14T14:00:57,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=894, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857875) roll requested 2024-11-14T14:00:57,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857882 2024-11-14T14:00:57,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857882 2024-11-14T14:00:57,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,888 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=895, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857882) roll requested 2024-11-14T14:00:57,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857888 2024-11-14T14:00:57,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857888 2024-11-14T14:00:57,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,896 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=896, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,896 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857888) roll requested 2024-11-14T14:00:57,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857896 2024-11-14T14:00:57,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857896 2024-11-14T14:00:57,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,903 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=897, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,903 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857896) roll requested 2024-11-14T14:00:57,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857903 2024-11-14T14:00:57,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857903 2024-11-14T14:00:57,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,910 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=898, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,910 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857903) roll requested 2024-11-14T14:00:57,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857910 2024-11-14T14:00:57,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857903 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857910 2024-11-14T14:00:57,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,916 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=899, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857910) roll requested 2024-11-14T14:00:57,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857917 2024-11-14T14:00:57,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857917 2024-11-14T14:00:57,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=900, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857917) roll requested 2024-11-14T14:00:57,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857924 2024-11-14T14:00:57,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857924 2024-11-14T14:00:57,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=901, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857924) roll requested 2024-11-14T14:00:57,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857931 2024-11-14T14:00:57,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857931 2024-11-14T14:00:57,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=902, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857931) roll requested 2024-11-14T14:00:57,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857938 2024-11-14T14:00:57,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857938 2024-11-14T14:00:57,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=903, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857938) roll requested 2024-11-14T14:00:57,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857946 2024-11-14T14:00:57,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857946 2024-11-14T14:00:57,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=904, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857946) roll requested 2024-11-14T14:00:57,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857953 2024-11-14T14:00:57,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857953 2024-11-14T14:00:57,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=905, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857953) roll requested 2024-11-14T14:00:57,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857960 2024-11-14T14:00:57,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857960 2024-11-14T14:00:57,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=906, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857960) roll requested 2024-11-14T14:00:57,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857968 2024-11-14T14:00:57,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857968 2024-11-14T14:00:57,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=907, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,975 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857968) roll requested 2024-11-14T14:00:57,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857975 2024-11-14T14:00:57,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857975 2024-11-14T14:00:57,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,982 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,982 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=908, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,982 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857975) roll requested 2024-11-14T14:00:57,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857982 2024-11-14T14:00:57,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857982 2024-11-14T14:00:57,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=909, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857982) roll requested 2024-11-14T14:00:57,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857989 2024-11-14T14:00:57,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:57,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857982 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857989 2024-11-14T14:00:57,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:57,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:57,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=910, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:57,996 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:57,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857989) roll requested 2024-11-14T14:00:57,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592857996 2024-11-14T14:00:58,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857996 2024-11-14T14:00:58,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=911, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592857996) roll requested 2024-11-14T14:00:58,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858003 2024-11-14T14:00:58,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592857996 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858003 2024-11-14T14:00:58,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=912, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858003) roll requested 2024-11-14T14:00:58,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858011 2024-11-14T14:00:58,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858011 2024-11-14T14:00:58,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=913, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858011) roll requested 2024-11-14T14:00:58,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858018 2024-11-14T14:00:58,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858018 2024-11-14T14:00:58,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=914, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858018) roll requested 2024-11-14T14:00:58,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858025 2024-11-14T14:00:58,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858025 2024-11-14T14:00:58,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=915, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858025) roll requested 2024-11-14T14:00:58,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858033 2024-11-14T14:00:58,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858033 2024-11-14T14:00:58,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=916, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858033) roll requested 2024-11-14T14:00:58,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858041 2024-11-14T14:00:58,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858041 2024-11-14T14:00:58,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=917, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858041) roll requested 2024-11-14T14:00:58,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858048 2024-11-14T14:00:58,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858048 2024-11-14T14:00:58,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=918, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858048) roll requested 2024-11-14T14:00:58,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858056 2024-11-14T14:00:58,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858056 2024-11-14T14:00:58,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,063 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,063 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=919, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858056) roll requested 2024-11-14T14:00:58,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858063 2024-11-14T14:00:58,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858056 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858063 2024-11-14T14:00:58,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=920, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858063) roll requested 2024-11-14T14:00:58,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858070 2024-11-14T14:00:58,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858070 2024-11-14T14:00:58,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=921, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858070) roll requested 2024-11-14T14:00:58,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858077 2024-11-14T14:00:58,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858077 2024-11-14T14:00:58,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,089 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,089 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=922, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858077) roll requested 2024-11-14T14:00:58,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858089 2024-11-14T14:00:58,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858089 2024-11-14T14:00:58,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=923, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858089) roll requested 2024-11-14T14:00:58,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858097 2024-11-14T14:00:58,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858089 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858097 2024-11-14T14:00:58,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=924, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858097) roll requested 2024-11-14T14:00:58,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858105 2024-11-14T14:00:58,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858105 2024-11-14T14:00:58,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=925, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,116 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858105) roll requested 2024-11-14T14:00:58,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858116 2024-11-14T14:00:58,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858116 2024-11-14T14:00:58,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,123 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,123 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=926, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,123 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858116) roll requested 2024-11-14T14:00:58,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858124 2024-11-14T14:00:58,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858116 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858124 2024-11-14T14:00:58,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,130 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=927, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858124) roll requested 2024-11-14T14:00:58,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858131 2024-11-14T14:00:58,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858131 2024-11-14T14:00:58,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=928, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858131) roll requested 2024-11-14T14:00:58,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858138 2024-11-14T14:00:58,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858131 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858138 2024-11-14T14:00:58,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=929, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858138) roll requested 2024-11-14T14:00:58,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858145 2024-11-14T14:00:58,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858145 2024-11-14T14:00:58,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=930, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858145) roll requested 2024-11-14T14:00:58,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858155 2024-11-14T14:00:58,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858155 2024-11-14T14:00:58,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=931, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858155) roll requested 2024-11-14T14:00:58,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858161 2024-11-14T14:00:58,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858161 2024-11-14T14:00:58,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,169 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,169 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=932, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,169 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858161) roll requested 2024-11-14T14:00:58,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858169 2024-11-14T14:00:58,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858169 2024-11-14T14:00:58,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=933, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858169) roll requested 2024-11-14T14:00:58,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858176 2024-11-14T14:00:58,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858169 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858176 2024-11-14T14:00:58,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=934, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858176) roll requested 2024-11-14T14:00:58,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858183 2024-11-14T14:00:58,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858176 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858183 2024-11-14T14:00:58,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=935, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858183) roll requested 2024-11-14T14:00:58,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858190 2024-11-14T14:00:58,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858183 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858190 2024-11-14T14:00:58,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=936, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858190) roll requested 2024-11-14T14:00:58,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858197 2024-11-14T14:00:58,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858197 2024-11-14T14:00:58,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=937, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858197) roll requested 2024-11-14T14:00:58,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858204 2024-11-14T14:00:58,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858204 2024-11-14T14:00:58,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,212 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=938, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858204) roll requested 2024-11-14T14:00:58,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858212 2024-11-14T14:00:58,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858212 2024-11-14T14:00:58,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=939, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858212) roll requested 2024-11-14T14:00:58,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858219 2024-11-14T14:00:58,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858219 2024-11-14T14:00:58,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=940, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858219) roll requested 2024-11-14T14:00:58,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858226 2024-11-14T14:00:58,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858226 2024-11-14T14:00:58,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=941, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858226) roll requested 2024-11-14T14:00:58,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858233 2024-11-14T14:00:58,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858233 2024-11-14T14:00:58,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=942, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858233) roll requested 2024-11-14T14:00:58,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858241 2024-11-14T14:00:58,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858241 2024-11-14T14:00:58,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=943, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858241) roll requested 2024-11-14T14:00:58,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858247 2024-11-14T14:00:58,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858247 2024-11-14T14:00:58,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,254 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,254 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=944, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,254 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858247) roll requested 2024-11-14T14:00:58,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858254 2024-11-14T14:00:58,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858254 2024-11-14T14:00:58,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=945, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858254) roll requested 2024-11-14T14:00:58,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858261 2024-11-14T14:00:58,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858254 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858261 2024-11-14T14:00:58,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=946, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858261) roll requested 2024-11-14T14:00:58,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858268 2024-11-14T14:00:58,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858268 2024-11-14T14:00:58,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,275 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=947, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858268) roll requested 2024-11-14T14:00:58,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858276 2024-11-14T14:00:58,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858276 2024-11-14T14:00:58,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=948, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858276) roll requested 2024-11-14T14:00:58,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858283 2024-11-14T14:00:58,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858276 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858283 2024-11-14T14:00:58,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,290 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=949, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,290 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858283) roll requested 2024-11-14T14:00:58,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858290 2024-11-14T14:00:58,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858290 2024-11-14T14:00:58,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=950, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858290) roll requested 2024-11-14T14:00:58,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858297 2024-11-14T14:00:58,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858297 2024-11-14T14:00:58,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=951, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858297) roll requested 2024-11-14T14:00:58,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858304 2024-11-14T14:00:58,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858304 2024-11-14T14:00:58,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=952, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858304) roll requested 2024-11-14T14:00:58,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858312 2024-11-14T14:00:58,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858312 2024-11-14T14:00:58,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=953, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858312) roll requested 2024-11-14T14:00:58,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858318 2024-11-14T14:00:58,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858318 2024-11-14T14:00:58,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,325 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=954, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858318) roll requested 2024-11-14T14:00:58,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858326 2024-11-14T14:00:58,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858326 2024-11-14T14:00:58,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=955, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858326) roll requested 2024-11-14T14:00:58,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858333 2024-11-14T14:00:58,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858326 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858333 2024-11-14T14:00:58,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=956, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858333) roll requested 2024-11-14T14:00:58,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858341 2024-11-14T14:00:58,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858341 2024-11-14T14:00:58,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=957, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858341) roll requested 2024-11-14T14:00:58,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858348 2024-11-14T14:00:58,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858348 2024-11-14T14:00:58,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=958, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858348) roll requested 2024-11-14T14:00:58,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858355 2024-11-14T14:00:58,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858355 2024-11-14T14:00:58,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=959, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858355) roll requested 2024-11-14T14:00:58,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858362 2024-11-14T14:00:58,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858362 2024-11-14T14:00:58,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=960, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858362) roll requested 2024-11-14T14:00:58,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858368 2024-11-14T14:00:58,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858362 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858368 2024-11-14T14:00:58,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=961, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858368) roll requested 2024-11-14T14:00:58,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858375 2024-11-14T14:00:58,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858375 2024-11-14T14:00:58,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=962, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858375) roll requested 2024-11-14T14:00:58,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858381 2024-11-14T14:00:58,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858381 2024-11-14T14:00:58,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=963, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858381) roll requested 2024-11-14T14:00:58,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858388 2024-11-14T14:00:58,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858388 2024-11-14T14:00:58,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,395 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=964, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,395 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858388) roll requested 2024-11-14T14:00:58,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858395 2024-11-14T14:00:58,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858388 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858395 2024-11-14T14:00:58,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=965, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858395) roll requested 2024-11-14T14:00:58,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858402 2024-11-14T14:00:58,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858402 2024-11-14T14:00:58,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,409 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,409 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=966, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,410 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858402) roll requested 2024-11-14T14:00:58,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858410 2024-11-14T14:00:58,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858410 2024-11-14T14:00:58,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=967, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858410) roll requested 2024-11-14T14:00:58,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858416 2024-11-14T14:00:58,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858410 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858416 2024-11-14T14:00:58,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,422 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,422 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=968, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,422 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858416) roll requested 2024-11-14T14:00:58,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858422 2024-11-14T14:00:58,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858422 2024-11-14T14:00:58,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,429 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,429 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=969, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,429 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858422) roll requested 2024-11-14T14:00:58,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858429 2024-11-14T14:00:58,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858422 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858429 2024-11-14T14:00:58,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=970, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858429) roll requested 2024-11-14T14:00:58,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858436 2024-11-14T14:00:58,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858436 2024-11-14T14:00:58,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=971, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858436) roll requested 2024-11-14T14:00:58,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858446 2024-11-14T14:00:58,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858446 2024-11-14T14:00:58,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,454 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=972, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858446) roll requested 2024-11-14T14:00:58,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858454 2024-11-14T14:00:58,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858454 2024-11-14T14:00:58,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=973, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858454) roll requested 2024-11-14T14:00:58,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858461 2024-11-14T14:00:58,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858461 2024-11-14T14:00:58,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=974, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858461) roll requested 2024-11-14T14:00:58,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858472 2024-11-14T14:00:58,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858472 2024-11-14T14:00:58,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,480 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,480 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=975, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,480 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858472) roll requested 2024-11-14T14:00:58,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858480 2024-11-14T14:00:58,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858480 2024-11-14T14:00:58,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,487 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=976, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,487 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858480) roll requested 2024-11-14T14:00:58,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858487 2024-11-14T14:00:58,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858480 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858487 2024-11-14T14:00:58,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=977, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858487) roll requested 2024-11-14T14:00:58,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858493 2024-11-14T14:00:58,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858487 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858493 2024-11-14T14:00:58,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=978, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858493) roll requested 2024-11-14T14:00:58,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858501 2024-11-14T14:00:58,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858501 2024-11-14T14:00:58,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=979, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858501) roll requested 2024-11-14T14:00:58,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858508 2024-11-14T14:00:58,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858508 2024-11-14T14:00:58,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,515 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=980, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,515 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858508) roll requested 2024-11-14T14:00:58,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858515 2024-11-14T14:00:58,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858515 2024-11-14T14:00:58,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,521 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,522 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=981, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858515) roll requested 2024-11-14T14:00:58,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858522 2024-11-14T14:00:58,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858522 2024-11-14T14:00:58,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=982, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858522) roll requested 2024-11-14T14:00:58,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858530 2024-11-14T14:00:58,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858530 2024-11-14T14:00:58,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=983, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858530) roll requested 2024-11-14T14:00:58,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858537 2024-11-14T14:00:58,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858537 2024-11-14T14:00:58,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=984, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,545 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858537) roll requested 2024-11-14T14:00:58,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858545 2024-11-14T14:00:58,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858545 2024-11-14T14:00:58,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=985, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858545) roll requested 2024-11-14T14:00:58,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858552 2024-11-14T14:00:58,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858545 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858552 2024-11-14T14:00:58,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,559 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,559 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=986, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858552) roll requested 2024-11-14T14:00:58,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858559 2024-11-14T14:00:58,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858559 2024-11-14T14:00:58,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,566 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,566 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=987, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,566 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858559) roll requested 2024-11-14T14:00:58,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858566 2024-11-14T14:00:58,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858559 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858566 2024-11-14T14:00:58,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,574 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=988, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,574 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858566) roll requested 2024-11-14T14:00:58,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858574 2024-11-14T14:00:58,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858566 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858574 2024-11-14T14:00:58,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=989, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858574) roll requested 2024-11-14T14:00:58,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858581 2024-11-14T14:00:58,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858574 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858581 2024-11-14T14:00:58,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=990, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858581) roll requested 2024-11-14T14:00:58,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858589 2024-11-14T14:00:58,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858589 2024-11-14T14:00:58,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,595 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=991, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858589) roll requested 2024-11-14T14:00:58,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858596 2024-11-14T14:00:58,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858596 2024-11-14T14:00:58,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,602 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=992, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,603 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858596) roll requested 2024-11-14T14:00:58,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858603 2024-11-14T14:00:58,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858603 2024-11-14T14:00:58,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=993, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858603) roll requested 2024-11-14T14:00:58,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858610 2024-11-14T14:00:58,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858610 2024-11-14T14:00:58,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,616 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=994, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858610) roll requested 2024-11-14T14:00:58,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858617 2024-11-14T14:00:58,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858617 2024-11-14T14:00:58,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,623 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=995, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,623 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858617) roll requested 2024-11-14T14:00:58,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858623 2024-11-14T14:00:58,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858623 2024-11-14T14:00:58,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=996, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858623) roll requested 2024-11-14T14:00:58,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858630 2024-11-14T14:00:58,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858630 2024-11-14T14:00:58,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,636 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,636 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=997, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,636 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858630) roll requested 2024-11-14T14:00:58,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858637 2024-11-14T14:00:58,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858637 2024-11-14T14:00:58,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,643 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=998, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,644 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858637) roll requested 2024-11-14T14:00:58,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858644 2024-11-14T14:00:58,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858644 2024-11-14T14:00:58,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=999, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858644) roll requested 2024-11-14T14:00:58,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858651 2024-11-14T14:00:58,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858651 2024-11-14T14:00:58,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1000, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858651) roll requested 2024-11-14T14:00:58,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858659 2024-11-14T14:00:58,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858659 2024-11-14T14:00:58,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,666 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1001, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,666 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858659) roll requested 2024-11-14T14:00:58,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858666 2024-11-14T14:00:58,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858666 2024-11-14T14:00:58,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1002, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858666) roll requested 2024-11-14T14:00:58,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858673 2024-11-14T14:00:58,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858666 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858673 2024-11-14T14:00:58,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,680 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1003, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858673) roll requested 2024-11-14T14:00:58,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858681 2024-11-14T14:00:58,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858681 2024-11-14T14:00:58,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,688 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1004, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858681) roll requested 2024-11-14T14:00:58,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858688 2024-11-14T14:00:58,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858688 2024-11-14T14:00:58,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1005, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858688) roll requested 2024-11-14T14:00:58,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858696 2024-11-14T14:00:58,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858696 2024-11-14T14:00:58,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1006, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858696) roll requested 2024-11-14T14:00:58,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858703 2024-11-14T14:00:58,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858703 2024-11-14T14:00:58,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1007, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858703) roll requested 2024-11-14T14:00:58,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858709 2024-11-14T14:00:58,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858709 2024-11-14T14:00:58,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1008, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858709) roll requested 2024-11-14T14:00:58,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858716 2024-11-14T14:00:58,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858716 2024-11-14T14:00:58,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,723 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1009, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,723 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858716) roll requested 2024-11-14T14:00:58,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858723 2024-11-14T14:00:58,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858723 2024-11-14T14:00:58,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1010, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858723) roll requested 2024-11-14T14:00:58,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858730 2024-11-14T14:00:58,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858723 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858730 2024-11-14T14:00:58,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,737 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1011, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858730) roll requested 2024-11-14T14:00:58,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858737 2024-11-14T14:00:58,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858737 2024-11-14T14:00:58,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1012, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858737) roll requested 2024-11-14T14:00:58,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858745 2024-11-14T14:00:58,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858745 2024-11-14T14:00:58,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,751 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1013, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858745) roll requested 2024-11-14T14:00:58,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858752 2024-11-14T14:00:58,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858752 2024-11-14T14:00:58,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1014, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858752) roll requested 2024-11-14T14:00:58,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858759 2024-11-14T14:00:58,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858759 2024-11-14T14:00:58,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1015, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858759) roll requested 2024-11-14T14:00:58,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858766 2024-11-14T14:00:58,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858766 2024-11-14T14:00:58,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,775 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1016, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,775 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858766) roll requested 2024-11-14T14:00:58,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858775 2024-11-14T14:00:58,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858775 2024-11-14T14:00:58,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,782 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1017, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858775) roll requested 2024-11-14T14:00:58,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858782 2024-11-14T14:00:58,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858775 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858782 2024-11-14T14:00:58,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,789 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1018, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,789 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858782) roll requested 2024-11-14T14:00:58,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858789 2024-11-14T14:00:58,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858789 2024-11-14T14:00:58,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1019, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858789) roll requested 2024-11-14T14:00:58,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858796 2024-11-14T14:00:58,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858796 2024-11-14T14:00:58,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,803 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,803 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1020, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858796) roll requested 2024-11-14T14:00:58,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858803 2024-11-14T14:00:58,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858803 2024-11-14T14:00:58,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,810 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1021, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,810 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858803) roll requested 2024-11-14T14:00:58,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858810 2024-11-14T14:00:58,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858810 2024-11-14T14:00:58,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,817 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1022, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858810) roll requested 2024-11-14T14:00:58,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858817 2024-11-14T14:00:58,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858810 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858817 2024-11-14T14:00:58,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1023, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858817) roll requested 2024-11-14T14:00:58,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858824 2024-11-14T14:00:58,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858824 2024-11-14T14:00:58,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,832 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,832 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1024, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,832 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858824) roll requested 2024-11-14T14:00:58,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858832 2024-11-14T14:00:58,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858832 2024-11-14T14:00:58,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1025, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858832) roll requested 2024-11-14T14:00:58,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858840 2024-11-14T14:00:58,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858832 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858840 2024-11-14T14:00:58,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1026, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858840) roll requested 2024-11-14T14:00:58,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858847 2024-11-14T14:00:58,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858847 2024-11-14T14:00:58,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1027, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858847) roll requested 2024-11-14T14:00:58,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858854 2024-11-14T14:00:58,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858854 2024-11-14T14:00:58,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1028, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,861 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858854) roll requested 2024-11-14T14:00:58,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858861 2024-11-14T14:00:58,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858861 2024-11-14T14:00:58,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,869 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1029, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,869 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858861) roll requested 2024-11-14T14:00:58,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858869 2024-11-14T14:00:58,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858869 2024-11-14T14:00:58,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,876 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,876 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1030, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,876 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858869) roll requested 2024-11-14T14:00:58,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858876 2024-11-14T14:00:58,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858869 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858876 2024-11-14T14:00:58,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,883 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1031, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,883 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858876) roll requested 2024-11-14T14:00:58,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858883 2024-11-14T14:00:58,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858883 2024-11-14T14:00:58,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1032, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858883) roll requested 2024-11-14T14:00:58,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858890 2024-11-14T14:00:58,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858890 2024-11-14T14:00:58,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1033, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,898 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858890) roll requested 2024-11-14T14:00:58,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858898 2024-11-14T14:00:58,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858898 2024-11-14T14:00:58,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,904 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1034, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858898) roll requested 2024-11-14T14:00:58,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858905 2024-11-14T14:00:58,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858898 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858905 2024-11-14T14:00:58,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1035, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858905) roll requested 2024-11-14T14:00:58,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858912 2024-11-14T14:00:58,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858912 2024-11-14T14:00:58,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1036, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858912) roll requested 2024-11-14T14:00:58,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858921 2024-11-14T14:00:58,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858921 2024-11-14T14:00:58,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,934 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1037, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858921) roll requested 2024-11-14T14:00:58,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858934 2024-11-14T14:00:58,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858934 2024-11-14T14:00:58,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,942 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1038, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,942 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858934) roll requested 2024-11-14T14:00:58,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858942 2024-11-14T14:00:58,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858942 2024-11-14T14:00:58,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,949 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1039, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,950 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858942) roll requested 2024-11-14T14:00:58,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858950 2024-11-14T14:00:58,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858942 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858950 2024-11-14T14:00:58,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1040, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,957 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858950) roll requested 2024-11-14T14:00:58,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858957 2024-11-14T14:00:58,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858950 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858957 2024-11-14T14:00:58,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1041, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858957) roll requested 2024-11-14T14:00:58,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858964 2024-11-14T14:00:58,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858957 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858964 2024-11-14T14:00:58,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1042, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858964) roll requested 2024-11-14T14:00:58,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858971 2024-11-14T14:00:58,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858971 2024-11-14T14:00:58,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1043, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858971) roll requested 2024-11-14T14:00:58,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858978 2024-11-14T14:00:58,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858978 2024-11-14T14:00:58,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1044, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858978) roll requested 2024-11-14T14:00:58,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858985 2024-11-14T14:00:58,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858985 2024-11-14T14:00:58,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1045, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858985) roll requested 2024-11-14T14:00:58,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592858992 2024-11-14T14:00:58,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:58,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858992 2024-11-14T14:00:58,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:58,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:58,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1046, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:58,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:58,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592858992) roll requested 2024-11-14T14:00:59,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859000 2024-11-14T14:00:59,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592858992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859000 2024-11-14T14:00:59,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1047, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859000) roll requested 2024-11-14T14:00:59,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859006 2024-11-14T14:00:59,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859006 2024-11-14T14:00:59,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1048, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859006) roll requested 2024-11-14T14:00:59,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859014 2024-11-14T14:00:59,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859014 2024-11-14T14:00:59,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1049, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859014) roll requested 2024-11-14T14:00:59,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859021 2024-11-14T14:00:59,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859021 2024-11-14T14:00:59,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1050, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859021) roll requested 2024-11-14T14:00:59,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859028 2024-11-14T14:00:59,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859028 2024-11-14T14:00:59,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,035 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,035 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1051, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,035 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859028) roll requested 2024-11-14T14:00:59,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859035 2024-11-14T14:00:59,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859035 2024-11-14T14:00:59,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,042 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1052, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859035) roll requested 2024-11-14T14:00:59,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859042 2024-11-14T14:00:59,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859042 2024-11-14T14:00:59,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,049 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1053, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859042) roll requested 2024-11-14T14:00:59,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859049 2024-11-14T14:00:59,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859042 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859049 2024-11-14T14:00:59,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,056 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1054, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859049) roll requested 2024-11-14T14:00:59,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859057 2024-11-14T14:00:59,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859057 2024-11-14T14:00:59,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1055, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,064 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859057) roll requested 2024-11-14T14:00:59,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859064 2024-11-14T14:00:59,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859064 2024-11-14T14:00:59,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,071 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,071 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1056, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,071 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859064) roll requested 2024-11-14T14:00:59,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859071 2024-11-14T14:00:59,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859064 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859071 2024-11-14T14:00:59,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1057, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859071) roll requested 2024-11-14T14:00:59,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859078 2024-11-14T14:00:59,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859078 2024-11-14T14:00:59,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1058, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859078) roll requested 2024-11-14T14:00:59,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859085 2024-11-14T14:00:59,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859085 2024-11-14T14:00:59,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1059, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859085) roll requested 2024-11-14T14:00:59,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859093 2024-11-14T14:00:59,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859093 2024-11-14T14:00:59,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,099 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1060, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,099 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859093) roll requested 2024-11-14T14:00:59,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859099 2024-11-14T14:00:59,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859099 2024-11-14T14:00:59,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,106 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,106 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1061, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,106 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859099) roll requested 2024-11-14T14:00:59,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859106 2024-11-14T14:00:59,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859099 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859106 2024-11-14T14:00:59,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1062, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859106) roll requested 2024-11-14T14:00:59,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859113 2024-11-14T14:00:59,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859106 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859113 2024-11-14T14:00:59,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1063, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859113) roll requested 2024-11-14T14:00:59,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859119 2024-11-14T14:00:59,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859119 2024-11-14T14:00:59,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,126 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1064, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,127 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859119) roll requested 2024-11-14T14:00:59,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859127 2024-11-14T14:00:59,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859127 2024-11-14T14:00:59,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1065, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859127) roll requested 2024-11-14T14:00:59,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859133 2024-11-14T14:00:59,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859127 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859133 2024-11-14T14:00:59,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1066, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859133) roll requested 2024-11-14T14:00:59,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859140 2024-11-14T14:00:59,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859140 2024-11-14T14:00:59,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,147 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1067, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,147 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859140) roll requested 2024-11-14T14:00:59,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859147 2024-11-14T14:00:59,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859147 2024-11-14T14:00:59,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1068, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859147) roll requested 2024-11-14T14:00:59,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859155 2024-11-14T14:00:59,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859147 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859155 2024-11-14T14:00:59,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1069, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859155) roll requested 2024-11-14T14:00:59,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859162 2024-11-14T14:00:59,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859162 2024-11-14T14:00:59,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1070, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859162) roll requested 2024-11-14T14:00:59,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859170 2024-11-14T14:00:59,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859170 2024-11-14T14:00:59,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1071, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859170) roll requested 2024-11-14T14:00:59,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859177 2024-11-14T14:00:59,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859170 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859177 2024-11-14T14:00:59,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1072, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859177) roll requested 2024-11-14T14:00:59,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859185 2024-11-14T14:00:59,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859185 2024-11-14T14:00:59,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,191 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,191 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1073, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,191 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859185) roll requested 2024-11-14T14:00:59,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859191 2024-11-14T14:00:59,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859191 2024-11-14T14:00:59,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1074, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859191) roll requested 2024-11-14T14:00:59,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859201 2024-11-14T14:00:59,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859191 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859201 2024-11-14T14:00:59,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,212 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,212 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1075, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859201) roll requested 2024-11-14T14:00:59,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859212 2024-11-14T14:00:59,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859212 2024-11-14T14:00:59,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1076, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859212) roll requested 2024-11-14T14:00:59,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859219 2024-11-14T14:00:59,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859219 2024-11-14T14:00:59,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1077, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859219) roll requested 2024-11-14T14:00:59,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859226 2024-11-14T14:00:59,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859226 2024-11-14T14:00:59,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1078, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859226) roll requested 2024-11-14T14:00:59,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859233 2024-11-14T14:00:59,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859233 2024-11-14T14:00:59,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1079, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859233) roll requested 2024-11-14T14:00:59,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859240 2024-11-14T14:00:59,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859240 2024-11-14T14:00:59,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1080, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859240) roll requested 2024-11-14T14:00:59,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859247 2024-11-14T14:00:59,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859247 2024-11-14T14:00:59,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,254 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1081, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859247) roll requested 2024-11-14T14:00:59,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859255 2024-11-14T14:00:59,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859255 2024-11-14T14:00:59,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,262 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1082, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,262 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859255) roll requested 2024-11-14T14:00:59,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859262 2024-11-14T14:00:59,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859262 2024-11-14T14:00:59,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1083, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859262) roll requested 2024-11-14T14:00:59,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859269 2024-11-14T14:00:59,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859262 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859269 2024-11-14T14:00:59,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,275 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1084, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859269) roll requested 2024-11-14T14:00:59,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859276 2024-11-14T14:00:59,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859276 2024-11-14T14:00:59,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1085, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859276) roll requested 2024-11-14T14:00:59,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859282 2024-11-14T14:00:59,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859276 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859282 2024-11-14T14:00:59,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,288 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,288 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1086, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,288 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859282) roll requested 2024-11-14T14:00:59,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859289 2024-11-14T14:00:59,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859289 2024-11-14T14:00:59,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,295 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1087, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859289) roll requested 2024-11-14T14:00:59,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859296 2024-11-14T14:00:59,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859296 2024-11-14T14:00:59,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1088, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859296) roll requested 2024-11-14T14:00:59,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859303 2024-11-14T14:00:59,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859303 2024-11-14T14:00:59,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,310 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1089, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,310 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859303) roll requested 2024-11-14T14:00:59,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859310 2024-11-14T14:00:59,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859310 2024-11-14T14:00:59,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1090, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859310) roll requested 2024-11-14T14:00:59,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859316 2024-11-14T14:00:59,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859310 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859316 2024-11-14T14:00:59,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1091, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859316) roll requested 2024-11-14T14:00:59,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859324 2024-11-14T14:00:59,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859324 2024-11-14T14:00:59,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1092, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,332 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859324) roll requested 2024-11-14T14:00:59,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859332 2024-11-14T14:00:59,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859332 2024-11-14T14:00:59,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1093, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859332) roll requested 2024-11-14T14:00:59,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859361 2024-11-14T14:00:59,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859361 2024-11-14T14:00:59,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1094, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859361) roll requested 2024-11-14T14:00:59,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859370 2024-11-14T14:00:59,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859370 2024-11-14T14:00:59,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1095, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859370) roll requested 2024-11-14T14:00:59,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859377 2024-11-14T14:00:59,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859377 2024-11-14T14:00:59,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1096, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859377) roll requested 2024-11-14T14:00:59,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859385 2024-11-14T14:00:59,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859385 2024-11-14T14:00:59,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1097, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859385) roll requested 2024-11-14T14:00:59,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859397 2024-11-14T14:00:59,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859397 2024-11-14T14:00:59,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1098, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859397) roll requested 2024-11-14T14:00:59,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859405 2024-11-14T14:00:59,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859405 2024-11-14T14:00:59,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,413 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,413 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1099, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,413 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859405) roll requested 2024-11-14T14:00:59,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859414 2024-11-14T14:00:59,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859414 2024-11-14T14:00:59,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1100, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859414) roll requested 2024-11-14T14:00:59,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859420 2024-11-14T14:00:59,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859420 2024-11-14T14:00:59,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1101, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859420) roll requested 2024-11-14T14:00:59,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859427 2024-11-14T14:00:59,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859427 2024-11-14T14:00:59,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,433 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,434 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1102, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859427) roll requested 2024-11-14T14:00:59,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859434 2024-11-14T14:00:59,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859434 2024-11-14T14:00:59,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1103, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859434) roll requested 2024-11-14T14:00:59,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859442 2024-11-14T14:00:59,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859442 2024-11-14T14:00:59,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1104, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,449 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859442) roll requested 2024-11-14T14:00:59,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859449 2024-11-14T14:00:59,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859449 2024-11-14T14:00:59,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,456 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1105, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,456 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859449) roll requested 2024-11-14T14:00:59,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859456 2024-11-14T14:00:59,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859449 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859456 2024-11-14T14:00:59,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,462 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1106, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859456) roll requested 2024-11-14T14:00:59,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859462 2024-11-14T14:00:59,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859462 2024-11-14T14:00:59,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1107, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859462) roll requested 2024-11-14T14:00:59,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859469 2024-11-14T14:00:59,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859469 2024-11-14T14:00:59,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,476 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1108, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,476 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859469) roll requested 2024-11-14T14:00:59,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859476 2024-11-14T14:00:59,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859476 2024-11-14T14:00:59,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1109, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859476) roll requested 2024-11-14T14:00:59,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859483 2024-11-14T14:00:59,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859483 2024-11-14T14:00:59,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1110, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859483) roll requested 2024-11-14T14:00:59,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859490 2024-11-14T14:00:59,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859490 2024-11-14T14:00:59,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1111, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,496 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859490) roll requested 2024-11-14T14:00:59,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859496 2024-11-14T14:00:59,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859490 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859496 2024-11-14T14:00:59,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1112, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859496) roll requested 2024-11-14T14:00:59,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859504 2024-11-14T14:00:59,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859496 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859504 2024-11-14T14:00:59,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1113, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859504) roll requested 2024-11-14T14:00:59,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859511 2024-11-14T14:00:59,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859511 2024-11-14T14:00:59,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,517 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,517 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1114, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859511) roll requested 2024-11-14T14:00:59,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859517 2024-11-14T14:00:59,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859517 2024-11-14T14:00:59,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1115, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859517) roll requested 2024-11-14T14:00:59,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859524 2024-11-14T14:00:59,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859524 2024-11-14T14:00:59,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1116, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859524) roll requested 2024-11-14T14:00:59,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859532 2024-11-14T14:00:59,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859532 2024-11-14T14:00:59,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,539 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,540 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1117, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,540 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859532) roll requested 2024-11-14T14:00:59,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859540 2024-11-14T14:00:59,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859540 2024-11-14T14:00:59,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1118, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859540) roll requested 2024-11-14T14:00:59,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859547 2024-11-14T14:00:59,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859540 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859547 2024-11-14T14:00:59,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,554 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1119, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,554 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859547) roll requested 2024-11-14T14:00:59,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859555 2024-11-14T14:00:59,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859555 2024-11-14T14:00:59,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1120, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859555) roll requested 2024-11-14T14:00:59,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859564 2024-11-14T14:00:59,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859564 2024-11-14T14:00:59,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1121, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,571 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859564) roll requested 2024-11-14T14:00:59,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859571 2024-11-14T14:00:59,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859571 2024-11-14T14:00:59,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,578 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,578 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1122, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,579 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859571) roll requested 2024-11-14T14:00:59,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859579 2024-11-14T14:00:59,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859579 2024-11-14T14:00:59,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,585 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1123, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,586 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859579) roll requested 2024-11-14T14:00:59,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859586 2024-11-14T14:00:59,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859586 2024-11-14T14:00:59,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1124, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859586) roll requested 2024-11-14T14:00:59,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859593 2024-11-14T14:00:59,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859586 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859593 2024-11-14T14:00:59,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1125, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859593) roll requested 2024-11-14T14:00:59,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859600 2024-11-14T14:00:59,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859600 2024-11-14T14:00:59,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,606 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1126, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859600) roll requested 2024-11-14T14:00:59,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859607 2024-11-14T14:00:59,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859607 2024-11-14T14:00:59,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,613 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,614 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1127, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,614 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859607) roll requested 2024-11-14T14:00:59,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859614 2024-11-14T14:00:59,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859614 2024-11-14T14:00:59,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1128, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859614) roll requested 2024-11-14T14:00:59,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859621 2024-11-14T14:00:59,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859621 2024-11-14T14:00:59,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1129, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859621) roll requested 2024-11-14T14:00:59,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859628 2024-11-14T14:00:59,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859628 2024-11-14T14:00:59,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1130, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859628) roll requested 2024-11-14T14:00:59,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859635 2024-11-14T14:00:59,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859635 2024-11-14T14:00:59,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,643 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1131, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,643 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859635) roll requested 2024-11-14T14:00:59,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859643 2024-11-14T14:00:59,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859643 2024-11-14T14:00:59,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,650 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,650 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1132, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,650 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859643) roll requested 2024-11-14T14:00:59,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859651 2024-11-14T14:00:59,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859643 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859651 2024-11-14T14:00:59,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1133, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859651) roll requested 2024-11-14T14:00:59,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859658 2024-11-14T14:00:59,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859658 2024-11-14T14:00:59,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1134, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859658) roll requested 2024-11-14T14:00:59,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859665 2024-11-14T14:00:59,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859665 2024-11-14T14:00:59,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1135, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859665) roll requested 2024-11-14T14:00:59,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859673 2024-11-14T14:00:59,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859673 2024-11-14T14:00:59,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1136, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859673) roll requested 2024-11-14T14:00:59,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859681 2024-11-14T14:00:59,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859681 2024-11-14T14:00:59,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1137, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859681) roll requested 2024-11-14T14:00:59,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859689 2024-11-14T14:00:59,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859689 2024-11-14T14:00:59,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1138, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859689) roll requested 2024-11-14T14:00:59,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859695 2024-11-14T14:00:59,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859695 2024-11-14T14:00:59,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1139, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859695) roll requested 2024-11-14T14:00:59,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859703 2024-11-14T14:00:59,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859703 2024-11-14T14:00:59,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,713 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1140, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,713 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859703) roll requested 2024-11-14T14:00:59,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859713 2024-11-14T14:00:59,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859713 2024-11-14T14:00:59,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1141, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859713) roll requested 2024-11-14T14:00:59,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859720 2024-11-14T14:00:59,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859713 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859720 2024-11-14T14:00:59,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,727 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1142, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859720) roll requested 2024-11-14T14:00:59,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859728 2024-11-14T14:00:59,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859720 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859728 2024-11-14T14:00:59,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,735 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1143, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859728) roll requested 2024-11-14T14:00:59,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859735 2024-11-14T14:00:59,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859735 2024-11-14T14:00:59,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,743 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1144, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,743 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859735) roll requested 2024-11-14T14:00:59,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859743 2024-11-14T14:00:59,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859743 2024-11-14T14:00:59,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1145, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859743) roll requested 2024-11-14T14:00:59,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859750 2024-11-14T14:00:59,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859750 2024-11-14T14:00:59,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1146, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859750) roll requested 2024-11-14T14:00:59,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859757 2024-11-14T14:00:59,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859750 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859757 2024-11-14T14:00:59,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,764 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1147, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,765 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859757) roll requested 2024-11-14T14:00:59,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859765 2024-11-14T14:00:59,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859765 2024-11-14T14:00:59,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1148, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,772 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859765) roll requested 2024-11-14T14:00:59,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859772 2024-11-14T14:00:59,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859765 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859772 2024-11-14T14:00:59,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1149, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859772) roll requested 2024-11-14T14:00:59,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859779 2024-11-14T14:00:59,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859779 2024-11-14T14:00:59,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,786 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1150, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859779) roll requested 2024-11-14T14:00:59,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859786 2024-11-14T14:00:59,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859786 2024-11-14T14:00:59,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1151, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859786) roll requested 2024-11-14T14:00:59,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859794 2024-11-14T14:00:59,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859794 2024-11-14T14:00:59,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1152, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859794) roll requested 2024-11-14T14:00:59,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859802 2024-11-14T14:00:59,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859802 2024-11-14T14:00:59,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,809 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1153, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859802) roll requested 2024-11-14T14:00:59,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859809 2024-11-14T14:00:59,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859809 2024-11-14T14:00:59,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1154, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859809) roll requested 2024-11-14T14:00:59,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859817 2024-11-14T14:00:59,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859817 2024-11-14T14:00:59,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1155, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859817) roll requested 2024-11-14T14:00:59,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859824 2024-11-14T14:00:59,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859824 2024-11-14T14:00:59,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,832 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1156, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,832 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859824) roll requested 2024-11-14T14:00:59,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859832 2024-11-14T14:00:59,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859832 2024-11-14T14:00:59,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1157, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859832) roll requested 2024-11-14T14:00:59,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859839 2024-11-14T14:00:59,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859832 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859839 2024-11-14T14:00:59,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1158, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859839) roll requested 2024-11-14T14:00:59,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859847 2024-11-14T14:00:59,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859847 2024-11-14T14:00:59,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1159, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859847) roll requested 2024-11-14T14:00:59,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859854 2024-11-14T14:00:59,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859854 2024-11-14T14:00:59,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1160, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,861 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859854) roll requested 2024-11-14T14:00:59,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859861 2024-11-14T14:00:59,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859861 2024-11-14T14:00:59,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,867 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1161, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,868 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859861) roll requested 2024-11-14T14:00:59,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859868 2024-11-14T14:00:59,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859868 2024-11-14T14:00:59,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,875 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1162, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859868) roll requested 2024-11-14T14:00:59,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859875 2024-11-14T14:00:59,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859868 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859875 2024-11-14T14:00:59,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,882 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1163, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,883 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859875) roll requested 2024-11-14T14:00:59,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859883 2024-11-14T14:00:59,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859883 2024-11-14T14:00:59,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1164, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859883) roll requested 2024-11-14T14:00:59,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859890 2024-11-14T14:00:59,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859890 2024-11-14T14:00:59,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1165, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859890) roll requested 2024-11-14T14:00:59,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859897 2024-11-14T14:00:59,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859897 2024-11-14T14:00:59,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,904 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,904 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1166, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859897) roll requested 2024-11-14T14:00:59,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859905 2024-11-14T14:00:59,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859905 2024-11-14T14:00:59,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1167, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859905) roll requested 2024-11-14T14:00:59,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859912 2024-11-14T14:00:59,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859912 2024-11-14T14:00:59,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1168, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,919 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859912) roll requested 2024-11-14T14:00:59,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859919 2024-11-14T14:00:59,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859919 2024-11-14T14:00:59,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,926 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1169, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859919) roll requested 2024-11-14T14:00:59,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859926 2024-11-14T14:00:59,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859919 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859926 2024-11-14T14:00:59,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,934 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,934 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1170, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859926) roll requested 2024-11-14T14:00:59,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859934 2024-11-14T14:00:59,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859926 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859934 2024-11-14T14:00:59,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1171, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859934) roll requested 2024-11-14T14:00:59,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859941 2024-11-14T14:00:59,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859941 2024-11-14T14:00:59,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1172, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859941) roll requested 2024-11-14T14:00:59,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859949 2024-11-14T14:00:59,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859949 2024-11-14T14:00:59,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1173, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859949) roll requested 2024-11-14T14:00:59,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859956 2024-11-14T14:00:59,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859956 2024-11-14T14:00:59,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,963 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,963 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1174, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,963 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859956) roll requested 2024-11-14T14:00:59,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859963 2024-11-14T14:00:59,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859963 2024-11-14T14:00:59,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,970 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,970 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1175, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,970 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859963) roll requested 2024-11-14T14:00:59,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859970 2024-11-14T14:00:59,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859963 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859970 2024-11-14T14:00:59,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1176, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859970) roll requested 2024-11-14T14:00:59,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859977 2024-11-14T14:00:59,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859970 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859977 2024-11-14T14:00:59,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1177, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859977) roll requested 2024-11-14T14:00:59,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859984 2024-11-14T14:00:59,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859984 2024-11-14T14:00:59,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1178, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859984) roll requested 2024-11-14T14:00:59,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859991 2024-11-14T14:00:59,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:00:59,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859991 2024-11-14T14:00:59,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:00:59,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:00:59,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1179, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:00:59,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:00:59,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859991) roll requested 2024-11-14T14:00:59,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592859999 2024-11-14T14:01:00,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859991 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859999 2024-11-14T14:01:00,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1180, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592859999) roll requested 2024-11-14T14:01:00,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860007 2024-11-14T14:01:00,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592859999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860007 2024-11-14T14:01:00,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1181, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860007) roll requested 2024-11-14T14:01:00,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860014 2024-11-14T14:01:00,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860014 2024-11-14T14:01:00,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1182, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860014) roll requested 2024-11-14T14:01:00,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860021 2024-11-14T14:01:00,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860021 2024-11-14T14:01:00,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1183, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860021) roll requested 2024-11-14T14:01:00,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860028 2024-11-14T14:01:00,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860028 2024-11-14T14:01:00,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,035 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,035 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1184, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860028) roll requested 2024-11-14T14:01:00,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860036 2024-11-14T14:01:00,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860036 2024-11-14T14:01:00,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1185, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860036) roll requested 2024-11-14T14:01:00,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860043 2024-11-14T14:01:00,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860043 2024-11-14T14:01:00,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,050 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1186, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,050 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860043) roll requested 2024-11-14T14:01:00,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860050 2024-11-14T14:01:00,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860050 2024-11-14T14:01:00,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1187, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860050) roll requested 2024-11-14T14:01:00,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860057 2024-11-14T14:01:00,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860057 2024-11-14T14:01:00,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,066 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1188, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860057) roll requested 2024-11-14T14:01:00,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860066 2024-11-14T14:01:00,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860066 2024-11-14T14:01:00,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1189, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860066) roll requested 2024-11-14T14:01:00,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860073 2024-11-14T14:01:00,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860073 2024-11-14T14:01:00,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1190, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860073) roll requested 2024-11-14T14:01:00,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860080 2024-11-14T14:01:00,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860080 2024-11-14T14:01:00,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,087 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,087 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1191, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860080) roll requested 2024-11-14T14:01:00,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860087 2024-11-14T14:01:00,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860087 2024-11-14T14:01:00,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,094 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,094 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1192, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,094 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860087) roll requested 2024-11-14T14:01:00,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860094 2024-11-14T14:01:00,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860094 2024-11-14T14:01:00,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1193, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860094) roll requested 2024-11-14T14:01:00,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860101 2024-11-14T14:01:00,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860094 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860101 2024-11-14T14:01:00,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1194, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860101) roll requested 2024-11-14T14:01:00,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860108 2024-11-14T14:01:00,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860108 2024-11-14T14:01:00,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1195, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860108) roll requested 2024-11-14T14:01:00,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860116 2024-11-14T14:01:00,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860116 2024-11-14T14:01:00,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,122 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,123 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1196, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,123 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860116) roll requested 2024-11-14T14:01:00,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860123 2024-11-14T14:01:00,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860116 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860123 2024-11-14T14:01:00,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1197, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860123) roll requested 2024-11-14T14:01:00,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860130 2024-11-14T14:01:00,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860123 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860130 2024-11-14T14:01:00,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1198, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,137 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860130) roll requested 2024-11-14T14:01:00,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860137 2024-11-14T14:01:00,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860137 2024-11-14T14:01:00,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1199, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860137) roll requested 2024-11-14T14:01:00,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860144 2024-11-14T14:01:00,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860137 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860144 2024-11-14T14:01:00,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,151 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,151 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1200, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,151 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860144) roll requested 2024-11-14T14:01:00,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860151 2024-11-14T14:01:00,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860151 2024-11-14T14:01:00,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,158 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,158 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1201, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,158 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860151) roll requested 2024-11-14T14:01:00,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860158 2024-11-14T14:01:00,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860158 2024-11-14T14:01:00,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1202, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860158) roll requested 2024-11-14T14:01:00,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860165 2024-11-14T14:01:00,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860158 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860165 2024-11-14T14:01:00,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,172 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1203, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,172 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860165) roll requested 2024-11-14T14:01:00,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860172 2024-11-14T14:01:00,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860172 2024-11-14T14:01:00,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1204, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860172) roll requested 2024-11-14T14:01:00,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860180 2024-11-14T14:01:00,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860172 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860180 2024-11-14T14:01:00,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,186 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,187 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1205, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,187 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860180) roll requested 2024-11-14T14:01:00,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860187 2024-11-14T14:01:00,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860187 2024-11-14T14:01:00,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,193 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,194 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1206, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,194 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860187) roll requested 2024-11-14T14:01:00,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860194 2024-11-14T14:01:00,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860187 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860194 2024-11-14T14:01:00,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1207, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860194) roll requested 2024-11-14T14:01:00,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860201 2024-11-14T14:01:00,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860194 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860201 2024-11-14T14:01:00,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1208, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860201) roll requested 2024-11-14T14:01:00,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860209 2024-11-14T14:01:00,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860209 2024-11-14T14:01:00,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1209, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860209) roll requested 2024-11-14T14:01:00,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860216 2024-11-14T14:01:00,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860216 2024-11-14T14:01:00,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1210, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860216) roll requested 2024-11-14T14:01:00,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860224 2024-11-14T14:01:00,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860224 2024-11-14T14:01:00,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1211, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860224) roll requested 2024-11-14T14:01:00,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860233 2024-11-14T14:01:00,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860224 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860233 2024-11-14T14:01:00,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1212, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860233) roll requested 2024-11-14T14:01:00,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860247 2024-11-14T14:01:00,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860247 2024-11-14T14:01:00,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1213, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860247) roll requested 2024-11-14T14:01:00,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860257 2024-11-14T14:01:00,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860257 2024-11-14T14:01:00,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1214, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860257) roll requested 2024-11-14T14:01:00,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860269 2024-11-14T14:01:00,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860269 2024-11-14T14:01:00,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1215, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860269) roll requested 2024-11-14T14:01:00,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860280 2024-11-14T14:01:00,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860280 2024-11-14T14:01:00,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,286 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1216, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,286 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860280) roll requested 2024-11-14T14:01:00,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860287 2024-11-14T14:01:00,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860287 2024-11-14T14:01:00,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1217, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860287) roll requested 2024-11-14T14:01:00,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860294 2024-11-14T14:01:00,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860287 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860294 2024-11-14T14:01:00,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1218, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860294) roll requested 2024-11-14T14:01:00,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860301 2024-11-14T14:01:00,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860301 2024-11-14T14:01:00,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1219, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860301) roll requested 2024-11-14T14:01:00,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860309 2024-11-14T14:01:00,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860309 2024-11-14T14:01:00,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1220, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860309) roll requested 2024-11-14T14:01:00,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860317 2024-11-14T14:01:00,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860317 2024-11-14T14:01:00,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1221, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860317) roll requested 2024-11-14T14:01:00,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860324 2024-11-14T14:01:00,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860324 2024-11-14T14:01:00,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,330 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,330 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1222, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,330 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860324) roll requested 2024-11-14T14:01:00,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860331 2024-11-14T14:01:00,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860331 2024-11-14T14:01:00,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,337 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1223, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,338 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860331) roll requested 2024-11-14T14:01:00,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860338 2024-11-14T14:01:00,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860338 2024-11-14T14:01:00,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1224, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860338) roll requested 2024-11-14T14:01:00,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860345 2024-11-14T14:01:00,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860338 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860345 2024-11-14T14:01:00,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,352 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1225, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860345) roll requested 2024-11-14T14:01:00,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860352 2024-11-14T14:01:00,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860352 2024-11-14T14:01:00,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,359 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1226, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,359 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860352) roll requested 2024-11-14T14:01:00,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860359 2024-11-14T14:01:00,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860359 2024-11-14T14:01:00,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,366 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,366 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1227, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,366 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860359) roll requested 2024-11-14T14:01:00,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860366 2024-11-14T14:01:00,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860359 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860366 2024-11-14T14:01:00,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,373 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1228, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860366) roll requested 2024-11-14T14:01:00,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860374 2024-11-14T14:01:00,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860366 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860374 2024-11-14T14:01:00,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,382 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1229, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,382 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860374) roll requested 2024-11-14T14:01:00,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860382 2024-11-14T14:01:00,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860382 2024-11-14T14:01:00,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,393 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,393 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1230, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860382) roll requested 2024-11-14T14:01:00,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860394 2024-11-14T14:01:00,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860382 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860394 2024-11-14T14:01:00,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1231, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860394) roll requested 2024-11-14T14:01:00,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860401 2024-11-14T14:01:00,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860401 2024-11-14T14:01:00,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,409 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1232, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860401) roll requested 2024-11-14T14:01:00,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860409 2024-11-14T14:01:00,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860409 2024-11-14T14:01:00,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1233, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860409) roll requested 2024-11-14T14:01:00,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860416 2024-11-14T14:01:00,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860416 2024-11-14T14:01:00,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1234, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860416) roll requested 2024-11-14T14:01:00,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860424 2024-11-14T14:01:00,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860424 2024-11-14T14:01:00,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1235, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860424) roll requested 2024-11-14T14:01:00,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860431 2024-11-14T14:01:00,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860431 2024-11-14T14:01:00,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,437 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,438 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1236, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860431) roll requested 2024-11-14T14:01:00,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860438 2024-11-14T14:01:00,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860438 2024-11-14T14:01:00,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,445 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,445 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1237, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860438) roll requested 2024-11-14T14:01:00,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860445 2024-11-14T14:01:00,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860445 2024-11-14T14:01:00,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,453 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1238, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,453 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860445) roll requested 2024-11-14T14:01:00,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860453 2024-11-14T14:01:00,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860453 2024-11-14T14:01:00,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1239, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,460 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860453) roll requested 2024-11-14T14:01:00,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860460 2024-11-14T14:01:00,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860453 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860460 2024-11-14T14:01:00,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1240, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860460) roll requested 2024-11-14T14:01:00,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860467 2024-11-14T14:01:00,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860460 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860467 2024-11-14T14:01:00,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1241, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860467) roll requested 2024-11-14T14:01:00,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860475 2024-11-14T14:01:00,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860475 2024-11-14T14:01:00,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,482 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,482 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1242, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,482 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860475) roll requested 2024-11-14T14:01:00,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860482 2024-11-14T14:01:00,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860482 2024-11-14T14:01:00,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1243, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,489 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860482) roll requested 2024-11-14T14:01:00,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860489 2024-11-14T14:01:00,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860489 2024-11-14T14:01:00,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1244, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,496 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860489) roll requested 2024-11-14T14:01:00,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860496 2024-11-14T14:01:00,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860496 2024-11-14T14:01:00,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1245, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860496) roll requested 2024-11-14T14:01:00,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860503 2024-11-14T14:01:00,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860496 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860503 2024-11-14T14:01:00,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,509 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1246, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,510 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860503) roll requested 2024-11-14T14:01:00,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860510 2024-11-14T14:01:00,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860510 2024-11-14T14:01:00,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,516 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,517 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1247, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860510) roll requested 2024-11-14T14:01:00,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860517 2024-11-14T14:01:00,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860510 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860517 2024-11-14T14:01:00,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1248, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860517) roll requested 2024-11-14T14:01:00,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860524 2024-11-14T14:01:00,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860524 2024-11-14T14:01:00,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1249, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860524) roll requested 2024-11-14T14:01:00,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860532 2024-11-14T14:01:00,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860532 2024-11-14T14:01:00,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,539 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,539 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1250, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,539 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860532) roll requested 2024-11-14T14:01:00,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860539 2024-11-14T14:01:00,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860539 2024-11-14T14:01:00,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,546 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1251, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,546 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860539) roll requested 2024-11-14T14:01:00,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860546 2024-11-14T14:01:00,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860539 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860546 2024-11-14T14:01:00,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,553 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1252, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,553 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860546) roll requested 2024-11-14T14:01:00,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860553 2024-11-14T14:01:00,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860546 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860553 2024-11-14T14:01:00,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,560 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,560 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1253, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,560 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860553) roll requested 2024-11-14T14:01:00,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860560 2024-11-14T14:01:00,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860560 2024-11-14T14:01:00,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,567 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,567 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1254, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,567 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860560) roll requested 2024-11-14T14:01:00,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860567 2024-11-14T14:01:00,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860560 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860567 2024-11-14T14:01:00,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,574 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,574 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1255, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,574 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860567) roll requested 2024-11-14T14:01:00,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860575 2024-11-14T14:01:00,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860567 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860575 2024-11-14T14:01:00,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1256, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860575) roll requested 2024-11-14T14:01:00,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860582 2024-11-14T14:01:00,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860582 2024-11-14T14:01:00,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1257, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860582) roll requested 2024-11-14T14:01:00,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860589 2024-11-14T14:01:00,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860589 2024-11-14T14:01:00,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1258, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860589) roll requested 2024-11-14T14:01:00,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860596 2024-11-14T14:01:00,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860596 2024-11-14T14:01:00,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,603 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1259, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,603 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860596) roll requested 2024-11-14T14:01:00,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860603 2024-11-14T14:01:00,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860603 2024-11-14T14:01:00,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1260, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860603) roll requested 2024-11-14T14:01:00,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860610 2024-11-14T14:01:00,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860610 2024-11-14T14:01:00,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1261, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860610) roll requested 2024-11-14T14:01:00,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860618 2024-11-14T14:01:00,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860618 2024-11-14T14:01:00,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1262, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860618) roll requested 2024-11-14T14:01:00,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860625 2024-11-14T14:01:00,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860625 2024-11-14T14:01:00,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,631 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,631 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1263, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,631 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860625) roll requested 2024-11-14T14:01:00,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860631 2024-11-14T14:01:00,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860631 2024-11-14T14:01:00,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1264, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,643 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860631) roll requested 2024-11-14T14:01:00,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860643 2024-11-14T14:01:00,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860631 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860643 2024-11-14T14:01:00,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,649 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,650 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1265, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,650 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860643) roll requested 2024-11-14T14:01:00,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860650 2024-11-14T14:01:00,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860643 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860650 2024-11-14T14:01:00,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1266, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860650) roll requested 2024-11-14T14:01:00,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860657 2024-11-14T14:01:00,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860650 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860657 2024-11-14T14:01:00,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1267, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860657) roll requested 2024-11-14T14:01:00,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860665 2024-11-14T14:01:00,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860665 2024-11-14T14:01:00,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1268, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860665) roll requested 2024-11-14T14:01:00,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860673 2024-11-14T14:01:00,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860673 2024-11-14T14:01:00,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,680 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1269, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,680 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860673) roll requested 2024-11-14T14:01:00,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860680 2024-11-14T14:01:00,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860680 2024-11-14T14:01:00,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1270, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860680) roll requested 2024-11-14T14:01:00,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860687 2024-11-14T14:01:00,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860680 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860687 2024-11-14T14:01:00,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1271, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860687) roll requested 2024-11-14T14:01:00,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860695 2024-11-14T14:01:00,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860695 2024-11-14T14:01:00,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1272, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860695) roll requested 2024-11-14T14:01:00,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860703 2024-11-14T14:01:00,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860703 2024-11-14T14:01:00,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,710 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1273, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,710 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860703) roll requested 2024-11-14T14:01:00,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860710 2024-11-14T14:01:00,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860710 2024-11-14T14:01:00,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,719 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1274, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,719 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860710) roll requested 2024-11-14T14:01:00,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860719 2024-11-14T14:01:00,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860710 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860719 2024-11-14T14:01:00,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,727 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1275, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,727 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860719) roll requested 2024-11-14T14:01:00,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860727 2024-11-14T14:01:00,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860719 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860727 2024-11-14T14:01:00,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1276, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860727) roll requested 2024-11-14T14:01:00,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860734 2024-11-14T14:01:00,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860727 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860734 2024-11-14T14:01:00,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,741 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1277, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860734) roll requested 2024-11-14T14:01:00,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860741 2024-11-14T14:01:00,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860741 2024-11-14T14:01:00,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1278, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860741) roll requested 2024-11-14T14:01:00,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860749 2024-11-14T14:01:00,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860749 2024-11-14T14:01:00,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1279, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860749) roll requested 2024-11-14T14:01:00,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860757 2024-11-14T14:01:00,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860757 2024-11-14T14:01:00,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,764 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,764 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1280, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,764 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860757) roll requested 2024-11-14T14:01:00,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860764 2024-11-14T14:01:00,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860764 2024-11-14T14:01:00,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1281, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,771 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860764) roll requested 2024-11-14T14:01:00,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860771 2024-11-14T14:01:00,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860764 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860771 2024-11-14T14:01:00,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1282, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860771) roll requested 2024-11-14T14:01:00,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860779 2024-11-14T14:01:00,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860771 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860779 2024-11-14T14:01:00,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,786 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1283, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860779) roll requested 2024-11-14T14:01:00,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860786 2024-11-14T14:01:00,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860786 2024-11-14T14:01:00,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1284, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860786) roll requested 2024-11-14T14:01:00,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860793 2024-11-14T14:01:00,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860793 2024-11-14T14:01:00,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1285, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860793) roll requested 2024-11-14T14:01:00,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860800 2024-11-14T14:01:00,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860800 2024-11-14T14:01:00,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,807 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,807 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1286, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860800) roll requested 2024-11-14T14:01:00,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860808 2024-11-14T14:01:00,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860808 2024-11-14T14:01:00,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1287, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860808) roll requested 2024-11-14T14:01:00,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860815 2024-11-14T14:01:00,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860815 2024-11-14T14:01:00,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1288, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860815) roll requested 2024-11-14T14:01:00,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860823 2024-11-14T14:01:00,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860823 2024-11-14T14:01:00,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1289, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860823) roll requested 2024-11-14T14:01:00,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860830 2024-11-14T14:01:00,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860830 2024-11-14T14:01:00,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1290, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860830) roll requested 2024-11-14T14:01:00,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860837 2024-11-14T14:01:00,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860837 2024-11-14T14:01:00,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,843 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,844 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1291, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,844 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860837) roll requested 2024-11-14T14:01:00,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860844 2024-11-14T14:01:00,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860844 2024-11-14T14:01:00,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,851 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,851 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1292, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,851 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860844) roll requested 2024-11-14T14:01:00,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860851 2024-11-14T14:01:00,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860844 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860851 2024-11-14T14:01:00,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,858 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1293, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860851) roll requested 2024-11-14T14:01:00,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860858 2024-11-14T14:01:00,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860858 2024-11-14T14:01:00,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1294, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860858) roll requested 2024-11-14T14:01:00,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860865 2024-11-14T14:01:00,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860865 2024-11-14T14:01:00,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1295, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860865) roll requested 2024-11-14T14:01:00,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860873 2024-11-14T14:01:00,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860873 2024-11-14T14:01:00,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1296, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860873) roll requested 2024-11-14T14:01:00,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860880 2024-11-14T14:01:00,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860880 2024-11-14T14:01:00,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1297, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860880) roll requested 2024-11-14T14:01:00,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860887 2024-11-14T14:01:00,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860887 2024-11-14T14:01:00,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,894 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,894 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1298, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860887) roll requested 2024-11-14T14:01:00,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860894 2024-11-14T14:01:00,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860894 2024-11-14T14:01:00,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1299, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860894) roll requested 2024-11-14T14:01:00,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860901 2024-11-14T14:01:00,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860901 2024-11-14T14:01:00,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1300, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860901) roll requested 2024-11-14T14:01:00,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860908 2024-11-14T14:01:00,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860908 2024-11-14T14:01:00,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,914 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,914 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1301, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,914 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860908) roll requested 2024-11-14T14:01:00,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860914 2024-11-14T14:01:00,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860908 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860914 2024-11-14T14:01:00,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,922 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1302, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,922 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860914) roll requested 2024-11-14T14:01:00,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860922 2024-11-14T14:01:00,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860922 2024-11-14T14:01:00,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,928 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1303, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,929 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860922) roll requested 2024-11-14T14:01:00,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860929 2024-11-14T14:01:00,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860922 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860929 2024-11-14T14:01:00,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1304, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860929) roll requested 2024-11-14T14:01:00,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860936 2024-11-14T14:01:00,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860936 2024-11-14T14:01:00,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,942 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1305, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860936) roll requested 2024-11-14T14:01:00,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860943 2024-11-14T14:01:00,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860943 2024-11-14T14:01:00,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,950 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,950 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1306, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,950 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860943) roll requested 2024-11-14T14:01:00,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860950 2024-11-14T14:01:00,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860950 2024-11-14T14:01:00,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,957 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1307, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,957 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860950) roll requested 2024-11-14T14:01:00,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860957 2024-11-14T14:01:00,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860950 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860957 2024-11-14T14:01:00,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,963 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1308, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860957) roll requested 2024-11-14T14:01:00,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860964 2024-11-14T14:01:00,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860957 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860964 2024-11-14T14:01:00,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,970 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1309, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860964) roll requested 2024-11-14T14:01:00,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860971 2024-11-14T14:01:00,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860971 2024-11-14T14:01:00,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1310, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860971) roll requested 2024-11-14T14:01:00,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860978 2024-11-14T14:01:00,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860978 2024-11-14T14:01:00,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1311, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860978) roll requested 2024-11-14T14:01:00,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860986 2024-11-14T14:01:00,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860986 2024-11-14T14:01:00,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1312, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:00,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:00,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860986) roll requested 2024-11-14T14:01:00,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592860993 2024-11-14T14:01:00,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:00,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860993 2024-11-14T14:01:00,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:00,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:00,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1313, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:01,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:01,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592860993) roll requested 2024-11-14T14:01:01,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592861000 2024-11-14T14:01:01,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592860993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861000 2024-11-14T14:01:01,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:01,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1314, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:01,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:01,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592861000) roll requested 2024-11-14T14:01:01,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592861007 2024-11-14T14:01:01,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861007 2024-11-14T14:01:01,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:01,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1315, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:01,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:01,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592861007) roll requested 2024-11-14T14:01:01,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592861014 2024-11-14T14:01:01,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861014 2024-11-14T14:01:01,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:01,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1316, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:01,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:01,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592861014) roll requested 2024-11-14T14:01:01,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592861021 2024-11-14T14:01:01,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861021 2024-11-14T14:01:01,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:01,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1317, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:01,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:01,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592861021) roll requested 2024-11-14T14:01:01,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592861029 2024-11-14T14:01:01,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861029 2024-11-14T14:01:01,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:01,035 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1318, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:01,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:01,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592861029) roll requested 2024-11-14T14:01:01,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592861036 2024-11-14T14:01:01,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861036 2024-11-14T14:01:01,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:01,043 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1319, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:01,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:01,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592861036) roll requested 2024-11-14T14:01:01,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592861043 2024-11-14T14:01:01,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861043 2024-11-14T14:01:01,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:01,050 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,050 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1320, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:01,050 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:01,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592861043) roll requested 2024-11-14T14:01:01,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592861050 2024-11-14T14:01:01,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861050 2024-11-14T14:01:01,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:01,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1321, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:01,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:01,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592861050) roll requested 2024-11-14T14:01:01,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592861058 2024-11-14T14:01:01,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861058 2024-11-14T14:01:01,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:01,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-14T14:01:01,065 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1322, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:01,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:01,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731592861058) roll requested 2024-11-14T14:01:01,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731592861065 2024-11-14T14:01:01,071 ERROR [Time-limited test {}] wal.AbstractFSWAL(939): Failed log archiving for the log /a/b, java.io.IOException: throw archival exception at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archiveLogFile(TestFailedAppendAndSync.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archive(AbstractFSWAL.java:934) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archive(TestFailedAppendAndSync.java:143) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync.testLockupAroundBadAssignSync(TestFailedAppendAndSync.java:280) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:01:01,073 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing fb54b5db78ce062cb48717f016e99f5f, disabling compactions & flushes 2024-11-14T14:01:01,073 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testLockupAroundBadAssignSync,,1731592830716.fb54b5db78ce062cb48717f016e99f5f. 2024-11-14T14:01:01,073 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testLockupAroundBadAssignSync,,1731592830716.fb54b5db78ce062cb48717f016e99f5f. 2024-11-14T14:01:01,074 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testLockupAroundBadAssignSync,,1731592830716.fb54b5db78ce062cb48717f016e99f5f. after waiting 0 ms 2024-11-14T14:01:01,074 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testLockupAroundBadAssignSync,,1731592830716.fb54b5db78ce062cb48717f016e99f5f. 2024-11-14T14:01:01,074 WARN [LogRoller {}] util.Shell(1106): Interrupted while joining on: Thread[Thread-8145,5,] java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Thread.join(Thread.java:1313) ~[?:?] at java.lang.Thread.join(Thread.java:1381) ~[?:?] at org.apache.hadoop.util.Shell.joinThread(Shell.java:1103) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1063) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1377) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1359) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:1116) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:454) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:414) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:566) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:597) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.(ChecksumFileSystem.java:637) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:721) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.createNonRecursive(ChecksumFileSystem.java:765) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem$FileSystemDataOutputStreamBuilder.build(FileSystem.java:4705) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:117) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:82) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.createWriterInstance(FSHLog.java:259) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:148) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriterInternal(AbstractFSWAL.java:1099) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$rollWriter$9(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.rollWriter(TestFailedAppendAndSync.java:129) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller$RollController.rollWal(AbstractWALRoller.java:311) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:212) ~[classes/:?] 2024-11-14T14:01:01,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,075 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 83 in region testLockupAroundBadAssignSync,,1731592830716.fb54b5db78ce062cb48717f016e99f5f. 2024-11-14T14:01:01,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,075 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testLockupAroundBadAssignSync,,1731592830716.fb54b5db78ce062cb48717f016e99f5f. 2024-11-14T14:01:01,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,076 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for fb54b5db78ce062cb48717f016e99f5f: Waiting for close lock at 1731592861073Disabling compacts and flushes for region at 1731592861073Disabling writes for close at 1731592861074 (+1 ms)Writing region close event to WAL at 1731592861075 (+1 ms)Closed at 1731592861075 2024-11-14T14:01:01,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861058 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731592861065 2024-11-14T14:01:01,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-14T14:01:01,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1323, max=55; forcing (partial) flush of 1 region(s): fb54b5db78ce062cb48717f016e99f5f[MyCF] 2024-11-14T14:01:01,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of fb54b5db78ce062cb48717f016e99f5f, because it is not online on us 2024-11-14T14:01:01,080 INFO [LogRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:01:01,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:01:01,452 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: DodgyFSLog wal:(num 1731592861065) 2024-11-14T14:01:01,452 INFO [Time-limited test {}] regionserver.TestFailedAppendAndSync(105): Cleaning test directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0e37b49f-6698-ee06-b593-f0a2a3d6670c 2024-11-14T14:01:02,090 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=22 (was 11) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=287 (was 287), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=362 (was 323) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6977 (was 7693)